From 4e96b7a930f5d7d24b242664e8e88b9b7801109c Mon Sep 17 00:00:00 2001 From: d1slike Date: Fri, 3 May 2024 17:46:22 +0300 Subject: [PATCH 01/40] Reimplement --- .gitignore | 8 +- .golangci.yml | 55 + .version | 2 +- CHANGELOG.md | 84 -- LICENSE | 674 ---------- README.md | 3 +- assembly/locator.go | 56 + cluster/client.go | 209 ---- cluster/const.go | 113 -- cluster/const_string.go | 36 - cluster/events.go | 79 -- cluster/leader_client.go | 119 -- codes/codes.go | 42 - conf/config.yml | 54 +- conf/config_test.yml | 7 - conf/configuration.go | 31 - ...onfig.json => default_remote_config.json} | 0 conf/local.go | 9 + conf/remote.go | 4 + controller/common_config.go | 123 -- controller/config.go | 236 ---- controller/helpers.go | 72 -- controller/module.go | 114 +- controller/route.go | 36 - controller/schema.go | 43 - docs/swagger.yaml | 1102 ----------------- domain/request.go | 35 - domain/response.go | 59 - entity/config.go | 46 - entity/module.go | 16 +- entity/version_config.go | 15 - go.mod | 167 ++- go.sum | 905 -------------- helper/handler.go | 68 - holder/holder.go | 14 - main.go | 268 +--- migrations/00001_2018-04-05_create_schema.sql | 5 - ...0002_2018-04-05_create_instances_table.sql | 47 - .../00003_2018-04-05_create_modules_table.sql | 24 - .../00004_2018-04-05_create_configs_table.sql | 70 -- .../00005_2018-07-31_add_name_for_configs.sql | 9 - .../00006_2018-07-31_config_sheme_table.sql | 23 - migrations/00007_2019-10-08_init_v2.sql | 46 - ...2020-06-03_create_version_config_table.sql | 13 - ...00009_2020-07-15_add_column_created_at.sql | 7 - migrations/20240503120009_init.sql | 55 + model/client.go | 23 - model/common_config.go | 55 - model/config.go | 55 - model/module.go | 54 - model/schema.go | 54 - model/version_config.go | 53 - raft/manager.go | 175 --- raft/network.go | 22 - repository/module.go | 22 + routes/routes.go | 46 + service/cluster_mesh.go | 28 - service/common_config.go | 91 -- service/config.go | 233 ---- service/discovery.go | 135 -- service/module.go | 4 + service/module_registry.go | 212 ---- service/room.go | 40 - service/routes.go | 54 - service/rqlite/db/db.go | 120 ++ service/rqlite/db/request.go | 5 + service/rqlite/db/result.go | 23 + service/rqlite/flags.go | 532 ++++++++ service/rqlite/goose_store/dialect.go | 37 + service/rqlite/goose_store/store.go | 105 ++ service/rqlite/main.go | 374 ++++++ service/rqlite/runner.go | 76 ++ service/schema.go | 36 - service/startup/service.go | 94 ++ service/version.go | 75 -- store/apply.go | 147 --- store/restore.go | 34 - store/state/common_config.go | 88 -- store/state/configs.go | 155 --- store/state/mesh.go | 95 -- store/state/modules.go | 78 -- store/state/schemas.go | 62 - store/state/state.go | 104 -- store/state/utils.go | 23 - store/state/version_config.go | 89 -- store/store.go | 137 -- subs/client.go | 124 -- subs/cluster.go | 31 - subs/conn.go | 31 - subs/helpers.go | 51 - subs/root.go | 118 -- 91 files changed, 1775 insertions(+), 7628 deletions(-) create mode 100644 .golangci.yml delete mode 100644 CHANGELOG.md delete mode 100644 LICENSE create mode 100644 assembly/locator.go delete mode 100644 cluster/client.go delete mode 100644 cluster/const.go delete mode 100644 cluster/const_string.go delete mode 100644 cluster/events.go delete mode 100644 cluster/leader_client.go delete mode 100644 codes/codes.go delete mode 100644 conf/config_test.yml delete mode 100644 conf/configuration.go rename conf/{default_remote_config.json => default_remote_config.json} (100%) create mode 100644 conf/local.go create mode 100644 conf/remote.go delete mode 100644 controller/common_config.go delete mode 100644 controller/config.go delete mode 100644 controller/helpers.go delete mode 100644 controller/route.go delete mode 100644 controller/schema.go delete mode 100644 docs/swagger.yaml delete mode 100644 domain/request.go delete mode 100644 domain/response.go delete mode 100644 entity/config.go delete mode 100644 entity/version_config.go delete mode 100644 go.sum delete mode 100644 helper/handler.go delete mode 100644 holder/holder.go delete mode 100644 migrations/00001_2018-04-05_create_schema.sql delete mode 100644 migrations/00002_2018-04-05_create_instances_table.sql delete mode 100644 migrations/00003_2018-04-05_create_modules_table.sql delete mode 100644 migrations/00004_2018-04-05_create_configs_table.sql delete mode 100644 migrations/00005_2018-07-31_add_name_for_configs.sql delete mode 100644 migrations/00006_2018-07-31_config_sheme_table.sql delete mode 100644 migrations/00007_2019-10-08_init_v2.sql delete mode 100644 migrations/00008_2020-06-03_create_version_config_table.sql delete mode 100644 migrations/00009_2020-07-15_add_column_created_at.sql create mode 100644 migrations/20240503120009_init.sql delete mode 100644 model/client.go delete mode 100644 model/common_config.go delete mode 100644 model/config.go delete mode 100644 model/module.go delete mode 100644 model/schema.go delete mode 100644 model/version_config.go delete mode 100644 raft/manager.go delete mode 100644 raft/network.go create mode 100644 repository/module.go create mode 100644 routes/routes.go delete mode 100644 service/cluster_mesh.go delete mode 100644 service/common_config.go delete mode 100644 service/config.go delete mode 100644 service/discovery.go create mode 100644 service/module.go delete mode 100644 service/module_registry.go delete mode 100644 service/room.go delete mode 100644 service/routes.go create mode 100644 service/rqlite/db/db.go create mode 100644 service/rqlite/db/request.go create mode 100644 service/rqlite/db/result.go create mode 100644 service/rqlite/flags.go create mode 100644 service/rqlite/goose_store/dialect.go create mode 100644 service/rqlite/goose_store/store.go create mode 100644 service/rqlite/main.go create mode 100644 service/rqlite/runner.go delete mode 100644 service/schema.go create mode 100644 service/startup/service.go delete mode 100644 service/version.go delete mode 100644 store/apply.go delete mode 100644 store/restore.go delete mode 100644 store/state/common_config.go delete mode 100644 store/state/configs.go delete mode 100644 store/state/mesh.go delete mode 100644 store/state/modules.go delete mode 100644 store/state/schemas.go delete mode 100644 store/state/state.go delete mode 100644 store/state/utils.go delete mode 100644 store/state/version_config.go delete mode 100644 store/store.go delete mode 100644 subs/client.go delete mode 100644 subs/cluster.go delete mode 100644 subs/conn.go delete mode 100644 subs/helpers.go delete mode 100644 subs/root.go diff --git a/.gitignore b/.gitignore index b91bded..26f2198 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,8 @@ .idea/**/tasks.xml .idea/dictionaries *.iml -config_service -config-service +convert_service +convert-service # Sensitive or high-churn files: .idea/**/dataSources/ @@ -69,3 +69,7 @@ fabric.properties /node_modules/ /vendor/ +package.json +yarn.lock + +conf/config_dev.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..cdb8788 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,55 @@ +run: + allow-parallel-runners: true +issues: + exclude-rules: + - path: controller #достаточное кол-во контекса на уровне контроллеров + linters: + - wrapcheck + - path: transaction #транзакции прозрачны + linters: + - wrapcheck +linters: + enable-all: true + disable: + - goimports #у нас собственные правила форматирования импортов + - wsl #расставление пробелов очень индивидуально + - exhaustivestruct #не имеет смысла + - varnamelen #очень индивидуально (db) + - exhaustruct #не имеет смысла + - golint #устарел + - contextcheck #не имеет смысла + - gci #у нас собственные правила форматирования импортов + - gofumpt #у нас собственные правила форматирования импортов + - nolintlint #Goland все равно форматирует с отступом + - nlreturn #не всегда имеет смысл + - godot #не актуально для свагера + - ifshort #приняли решение, что мы так не пишем if err := ...; err != nil {} + - godox #не позволяет оставлять todo + - maligned #аллокацией памяти для структур конфликтует с удобством их чтения + - nosnakecase #не позволяет заводить пакеты используя snake_case + - depguard #не всегда имеет смысл + - dupword #не имеет смысла + - tagalign #не всегда имеет смысл + - musttag #у нас собственные правило: не используем теги, если работаем с пакетом isp-kit/json и с внутренними структурами(между нашими сервисами), при этом теги обязательный для всех внених для нас структур + - perfsprint #мнимая производительность в угоду читаемости +linters-settings: + funlen: + lines: 80 + lll: + line-length: 150 + cyclop: + max-complexity: 15 #по умолчанию 10 мало + revive: + rules: + - name: var-naming #отключаем Id -> ID + disabled: true + stylecheck: + checks: #отключаем Id -> ID + - "-ST1000" + - "-ST1016" + - "-ST1020" + - "-ST1021" + - "-ST1022" + testifylint: + disable: + - compares diff --git a/.version b/.version index 3f5987a..4a36342 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.4.9 +3.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 83b80a2..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,84 +0,0 @@ -* исправлен метод /config/get_config_by_id -### v2.4.9 -* обновлены зависимости (исправляет panic при запуске) -### v2.4.8 -* do not check for instance_uuid anymore -### v2.4.7 -* updated dependencies -### v2.4.6 -* increase default version config count to 15 -* fix leader address race on leader disconnect (it was possible that after the change of the leader, one of the nodes will not be declared available, although this is not so) -* enable metric server -* add integration test for multiple config updates -* fix service hanging on shutdown -* improve raft logging -* fix deadlock in cluster client -* add timeouts to all websocket emits to prevent connections leakage -### v2.4.5 -* updated isp-lib -### v2.4.4 -* fix marshaling body of `ERROR_CONNECTION` event -### v2.4.3 -* updated isp-lib -* updated isp-event-lib -### v2.4.2 -* add db pass part -### v2.4.1 -* updated method `/config/get_configs_by_module_id` -### v2.4.0 -* add method `/config/get_config_by_id` -* add method `/config/get_configs_by_module_id` -* updated docs -### v2.3.1 -* fix old configs order -* add `createdAt` field to config version -### v2.3.0 -* add method `/module/broadcast_event` to broadcast arbitrary event to modules -* add module dependencies to response `/module/get_modules_info` -* add option to create a new config version instead update it -* fix broadcasting config on activating and upserting configs -* add saving old configs -* add method `/config/delete_version` -* add method `/config/get_all_version` -* update libs - -### v2.2.1 -* update deps -* fix bug in json schema marshaling/unmarshaling - -### v2.2.0 -* migrate to go modules -* add linter, refactor - -### v2.1.3 -* improved logging - -### v2.1.2 -* leader ws client ConnectionReadLimit is now configured by WS.WsConnectionReadLimitKB config param and defaults to 4 MB - -### v2.1.1 -* update to new endpoint descriptor struct with extra fields introduced in isp-lib v2.1.0 - -### v2.1.0 -* fix panic type casting in snapshot -* create empty default config now -* add method `/module/get_by_name` to fetch module object by name - -### v2.0.3 -* fix data race when applyCommandOnLeader in websocket handler -* update dependencies - -### v2.0.2 -* fix raft server address announcing (serverId == serverAddress == cluster.outerAddress) - -### v2.0.1 -* fix nil pointer dereference in repositories -* increase default websocket ConnectionReadLimit to 4 MB, add to configuration - -### v2.0.0 -* full rewrite to support cluster mode -* change websocket protocol from socketio to etp - -### v1.1.0 -* update libs -* default config handling diff --git a/LICENSE b/LICENSE deleted file mode 100644 index f288702..0000000 --- a/LICENSE +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/README.md b/README.md index 30d74d2..e9bccb5 100644 --- a/README.md +++ b/README.md @@ -1 +1,2 @@ -test \ No newline at end of file +# isp-config-service + diff --git a/assembly/locator.go b/assembly/locator.go new file mode 100644 index 0000000..55e9d26 --- /dev/null +++ b/assembly/locator.go @@ -0,0 +1,56 @@ +package assembly + +import ( + "net/http" + + "github.com/txix-open/etp/v3" + "github.com/txix-open/isp-kit/db" + "github.com/txix-open/isp-kit/grpc" + "github.com/txix-open/isp-kit/grpc/endpoint" + "github.com/txix-open/isp-kit/log" + "isp-config-service/routes" +) + +const ( + wsReadLimit = 4 * 1024 * 1024 +) + +type Locator struct { + db db.DB + logger log.Logger +} + +func NewLocator(logger log.Logger, db db.DB) Locator { + return Locator{ + db: db, + logger: logger, + } +} + +type Config struct { + grpcMux *grpc.Mux + httpMux http.Handler + etpSrv *etp.Server +} + +func (l Locator) Config() Config { + controllers := routes.Controllers{} + mapper := endpoint.DefaultWrapper(l.logger) + grpcMux := routes.GrpcHandler(mapper, controllers) + + etpSrv := etp.NewServer( + etp.WithServerReadLimit(wsReadLimit), + etp.WithServerAcceptOptions(&etp.AcceptOptions{ + InsecureSkipVerify: true, + }), + ) + routes.BindEtp(etpSrv, controllers) + + httpMux := routes.HttpHandler(etpSrv) + + return Config{ + grpcMux: grpcMux, + etpSrv: etpSrv, + httpMux: httpMux, + } +} diff --git a/cluster/client.go b/cluster/client.go deleted file mode 100644 index 5223d91..0000000 --- a/cluster/client.go +++ /dev/null @@ -1,209 +0,0 @@ -package cluster - -import ( - "errors" - "fmt" - "sync" - "time" - - "github.com/integration-system/isp-lib/v2/atomic" - "github.com/integration-system/isp-lib/v2/structure" - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - jsoniter "github.com/json-iterator/go" - "isp-config-service/codes" - "isp-config-service/entity" - "isp-config-service/raft" - "isp-config-service/store/state" -) - -var ( - ErrNoLeader = errors.New("no leader in cluster") - ErrLeaderClientNotInitialized = errors.New("leader client not initialized") - ErrNotLeader = errors.New("node is not a leader") - json = jsoniter.ConfigFastest -) - -const ( - leaderConnectionTimeout = 3 * time.Second - defaultApplyTimeout = 3 * time.Second -) - -type Client struct { - r *raft.Raft - - leaderMu sync.RWMutex - isLeader *atomic.AtomicBool - leaderState leaderState - leaderClient *SocketLeaderClient - declaration structure.BackendDeclaration - onClientDisconnect func(string) -} - -func (client *Client) Shutdown() error { - client.leaderMu.Lock() - defer client.leaderMu.Unlock() - - if client.leaderClient != nil { - client.leaderClient.Close() - client.leaderClient = nil - } - - return client.r.GracefulShutdown() -} - -func (client *Client) IsLeader() bool { - return client.isLeader.Get() -} - -func (client *Client) SyncApply(command []byte) (*ApplyLogResponse, error) { - client.leaderMu.RLock() - defer client.leaderMu.RUnlock() - - if !client.leaderState.leaderElected { - return nil, ErrNoLeader - } - - if client.leaderState.isLeader { - apply, err := client.r.SyncApply(command) - if err != nil { - return nil, fmt.Errorf("apply to raft as a leader: %v", err) - } - logResponse := apply.(ApplyLogResponse) - return &logResponse, nil - } - - if client.leaderClient == nil { - return nil, ErrLeaderClientNotInitialized - } - response, err := client.leaderClient.Ack(command, defaultApplyTimeout) - if err != nil { - return nil, fmt.Errorf("send to leader to apply: %v", err) - } - - var logResponse ApplyLogResponse - err = json.Unmarshal(response, &logResponse) - if err != nil { - return nil, err - } - - return &logResponse, nil -} - -func (client *Client) SyncApplyOnLeader(command []byte) (*ApplyLogResponse, error) { - client.leaderMu.RLock() - defer client.leaderMu.RUnlock() - - if !client.leaderState.isLeader { - return nil, ErrNotLeader - } - apply, err := client.r.SyncApply(command) - if err != nil { - return nil, fmt.Errorf("apply to raft as a leader: %v", err) - } - logResponse := apply.(ApplyLogResponse) - return &logResponse, nil -} - -func (client *Client) listenLeader() { - for n := range client.r.LeaderCh() { - client.leaderMu.Lock() - log.WithMetadata(map[string]interface{}{ - "leader_elected": n.LeaderElected, - "current_leader": n.CurrentLeaderAddress, - "is_leader": n.IsLeader, - }).Info(codes.LeaderStateChanged, "leader state changed") - - if client.leaderClient != nil { - client.leaderClient.Close() - client.leaderClient = nil - } - if n.LeaderElected && !n.IsLeader { - leaderAddr := n.CurrentLeaderAddress - leaderClient := NewSocketLeaderClient(leaderAddr, func() { - client.onClientDisconnect(leaderAddr) - }) - if err := leaderClient.Dial(leaderConnectionTimeout); err != nil { - log.Fatalf(codes.LeaderClientConnectionError, "could not connect to leader: %v", err) - } - client.leaderClient = leaderClient - - log.Info(codes.SendDeclarationToLeader, "sending declaration to leader through websocket") - go client.declareMyselfToLeader(leaderClient) - } else if n.LeaderElected && n.IsLeader { - go client.declareMyselfToCluster() - } - client.leaderState = leaderState{ - leaderElected: n.LeaderElected, - isLeader: n.IsLeader, - leaderAddr: n.CurrentLeaderAddress, - } - client.isLeader.Set(n.IsLeader) - - client.leaderMu.Unlock() - } -} -func (client *Client) declareMyselfToLeader(leaderClient *SocketLeaderClient) { - response, err := leaderClient.SendDeclaration(client.declaration, defaultApplyTimeout) - if err != nil { - log.Warnf(codes.SendDeclarationToLeader, "send declaration to leader. err: %v", err) - } else if response != utils.WsOkResponse { - log.Warnf(codes.SendDeclarationToLeader, "send declaration to leader. error response: '%s'", response) - } else { - log.Info(codes.SendDeclarationToLeader, "successfully sent my declaration to leader") - } -} - -// used when a leader need to declare himself to a cluster -func (client *Client) declareMyselfToCluster() { - now := state.GenerateDate() - module := entity.Module{ - Id: state.GenerateId(), - Name: client.declaration.ModuleName, - CreatedAt: now, - LastConnectedAt: now, - } - command := PrepareModuleConnectedCommand(module) - syncApplyCommand(client, command) - - declarationCommand := PrepareUpdateBackendDeclarationCommand(client.declaration) - syncApplyCommand(client, declarationCommand) -} - -type leaderState struct { - leaderElected bool - isLeader bool - leaderAddr string -} - -func NewRaftClusterClient(r *raft.Raft, declaration structure.BackendDeclaration, onLeaderDisconnect func(string)) *Client { - client := &Client{ - r: r, - declaration: declaration, - leaderState: leaderState{}, - onClientDisconnect: onLeaderDisconnect, - isLeader: atomic.NewAtomicBool(false), - } - go client.listenLeader() - - return client -} - -func syncApplyCommand(clusterClient *Client, command []byte) { - applyLogResponse, err := clusterClient.SyncApply(command) - if err != nil { - commandName := ParseCommand(command).String() - log.WithMetadata(map[string]interface{}{ - "command": string(command), - "commandName": commandName, - }).Warnf(codes.SyncApplyError, "announce myself. apply command: %v", err) - } - if applyLogResponse != nil && applyLogResponse.ApplyError != "" { - commandName := ParseCommand(command).String() - log.WithMetadata(map[string]interface{}{ - "result": string(applyLogResponse.Result), - "applyError": applyLogResponse.ApplyError, - "commandName": commandName, - }).Warnf(codes.SyncApplyError, "announce myself. apply command") - } -} diff --git a/cluster/const.go b/cluster/const.go deleted file mode 100644 index 9d1c6cc..0000000 --- a/cluster/const.go +++ /dev/null @@ -1,113 +0,0 @@ -package cluster - -import ( - "bytes" - "encoding/binary" - "time" - - "github.com/integration-system/isp-lib/v2/structure" - log "github.com/integration-system/isp-log" - "isp-config-service/codes" - "isp-config-service/entity" -) - -const ( - ApplyCommandEvent = "CONFIG_CLUSTER:APPLY_COMMAND" - CommandSizeBytes = 8 - - GETParam = "cluster" -) - -type Command uint64 - -// needed: go get golang.org/x/tools/cmd/stringer -//go:generate stringer -type=Command -output const_string.go - -const ( - _ Command = iota - UpdateBackendDeclarationCommand - DeleteBackendDeclarationCommand - - UpdateConfigSchemaCommand - - ModuleConnectedCommand - ModuleDisconnectedCommand - DeleteModulesCommand - DeleteVersionConfigCommand - - ActivateConfigCommand - DeleteConfigsCommand - UpsertConfigCommand - - DeleteCommonConfigsCommand - UpsertCommonConfigCommand - - BroadcastEventCommand -) - -func ParseCommand(data []byte) Command { - return Command(binary.BigEndian.Uint64(data[:CommandSizeBytes])) -} - -func PrepareUpdateBackendDeclarationCommand(backend structure.BackendDeclaration) []byte { - return prepareCommand(UpdateBackendDeclarationCommand, backend) -} - -func PrepareDeleteBackendDeclarationCommand(backend structure.BackendDeclaration) []byte { - return prepareCommand(DeleteBackendDeclarationCommand, backend) -} - -func PrepareModuleConnectedCommand(module entity.Module) []byte { - return prepareCommand(ModuleConnectedCommand, module) -} - -func PrepareModuleDisconnectedCommand(module entity.Module) []byte { - return prepareCommand(ModuleDisconnectedCommand, module) -} - -func PrepareDeleteModulesCommand(ids []string) []byte { - return prepareCommand(DeleteModulesCommand, DeleteModules{Ids: ids}) -} - -func PrepareActivateConfigCommand(configID string, date time.Time) []byte { - return prepareCommand(ActivateConfigCommand, ActivateConfig{ConfigId: configID, Date: date}) -} - -func PrepareDeleteConfigsCommand(ids []string) []byte { - return prepareCommand(DeleteConfigsCommand, DeleteModules{Ids: ids}) -} - -func PrepareUpsertConfigCommand(config UpsertConfig) []byte { - return prepareCommand(UpsertConfigCommand, config) -} - -func PrepareUpdateConfigSchemaCommand(schema entity.ConfigSchema) []byte { - return prepareCommand(UpdateConfigSchemaCommand, schema) -} - -func PrepareDeleteCommonConfigsCommand(id string) []byte { - return prepareCommand(DeleteCommonConfigsCommand, DeleteCommonConfig{Id: id}) -} - -func PrepareUpsertCommonConfigCommand(config UpsertCommonConfig) []byte { - return prepareCommand(UpsertCommonConfigCommand, config) -} - -func PrepareBroadcastEventCommand(event BroadcastEvent) []byte { - return prepareCommand(BroadcastEventCommand, event) -} - -func PrepareDeleteConfigVersionCommand(id string) []byte { - return prepareCommand(DeleteVersionConfigCommand, Identity{Id: id}) -} - -func prepareCommand(command Command, payload interface{}) []byte { - cmd := make([]byte, 8) - binary.BigEndian.PutUint64(cmd, uint64(command)) - buf := bytes.NewBuffer(cmd) - err := json.NewEncoder(buf).Encode(payload) - if err != nil { - log.Fatalf(codes.PrepareLogCommandError, "prepare log command: %v", err) - } - return buf.Bytes() -} diff --git a/cluster/const_string.go b/cluster/const_string.go deleted file mode 100644 index 07169c8..0000000 --- a/cluster/const_string.go +++ /dev/null @@ -1,36 +0,0 @@ -// Code generated by "stringer -type=Command -output const_string.go"; DO NOT EDIT. - -package cluster - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[UpdateBackendDeclarationCommand-1] - _ = x[DeleteBackendDeclarationCommand-2] - _ = x[UpdateConfigSchemaCommand-3] - _ = x[ModuleConnectedCommand-4] - _ = x[ModuleDisconnectedCommand-5] - _ = x[DeleteModulesCommand-6] - _ = x[DeleteVersionConfigCommand-7] - _ = x[ActivateConfigCommand-8] - _ = x[DeleteConfigsCommand-9] - _ = x[UpsertConfigCommand-10] - _ = x[DeleteCommonConfigsCommand-11] - _ = x[UpsertCommonConfigCommand-12] - _ = x[BroadcastEventCommand-13] -} - -const _Command_name = "UpdateBackendDeclarationCommandDeleteBackendDeclarationCommandUpdateConfigSchemaCommandModuleConnectedCommandModuleDisconnectedCommandDeleteModulesCommandDeleteVersionConfigCommandActivateConfigCommandDeleteConfigsCommandUpsertConfigCommandDeleteCommonConfigsCommandUpsertCommonConfigCommandBroadcastEventCommand" - -var _Command_index = [...]uint16{0, 31, 62, 87, 109, 134, 154, 180, 201, 221, 240, 266, 291, 312} - -func (i Command) String() string { - i -= 1 - if i >= Command(len(_Command_index)-1) { - return "Command(" + strconv.FormatInt(int64(i+1), 10) + ")" - } - return _Command_name[_Command_index[i]:_Command_index[i+1]] -} diff --git a/cluster/events.go b/cluster/events.go deleted file mode 100644 index 296cba8..0000000 --- a/cluster/events.go +++ /dev/null @@ -1,79 +0,0 @@ -package cluster - -import ( - json2 "encoding/json" - "fmt" - "time" - - "google.golang.org/grpc/codes" - "isp-config-service/entity" -) - -type ApplyLogResponse struct { - ApplyError string - Result json2.RawMessage -} - -func (a ApplyLogResponse) Error() string { - return a.ApplyError -} - -type DeleteModules struct { - Ids []string -} - -type ActivateConfig struct { - ConfigId string - Date time.Time -} - -type UpsertConfig struct { - Config entity.Config - VersionId string - VersionCreateAt time.Time - Create bool - Unsafe bool -} - -type DeleteConfigs struct { - Ids []string -} - -type DeleteCommonConfig struct { - Id string -} - -type Identity struct { - Id string -} - -type UpsertCommonConfig struct { - Config entity.CommonConfig - Create bool -} - -type BroadcastEvent struct { - Event string - ModuleNames []string - PerformUntil time.Time // stateless events, must be broadcast until this in UTC - Payload json2.RawMessage -} - -type ResponseWithError struct { - Response interface{} - Error string - ErrorCode codes.Code -} - -func NewResponseErrorf(code codes.Code, format string, args ...interface{}) ResponseWithError { - return ResponseWithError{ - Error: fmt.Sprintf(format, args...), - ErrorCode: code, - } -} - -func NewResponse(value interface{}) ResponseWithError { - return ResponseWithError{ - Response: value, - } -} diff --git a/cluster/leader_client.go b/cluster/leader_client.go deleted file mode 100644 index 4c412be..0000000 --- a/cluster/leader_client.go +++ /dev/null @@ -1,119 +0,0 @@ -package cluster - -import ( - "context" - "fmt" - "net" - "net/http" - "net/url" - "time" - - "github.com/cenkalti/backoff/v4" - etp "github.com/integration-system/isp-etp-go/v2/client" - "github.com/integration-system/isp-lib/v2/config" - "github.com/integration-system/isp-lib/v2/structure" - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - "isp-config-service/codes" - "isp-config-service/conf" -) - -var ( - // Set from config in main - WsConnectionReadLimit int64 -) - -type SocketLeaderClient struct { - client etp.Client - url string - globalCtx context.Context - cancel context.CancelFunc -} - -func (c *SocketLeaderClient) Ack(data []byte, timeout time.Duration) ([]byte, error) { - ctx, cancel := context.WithTimeout(c.globalCtx, timeout) - defer cancel() - response, err := c.client.EmitWithAck(ctx, ApplyCommandEvent, data) - return response, err -} - -func (c *SocketLeaderClient) SendDeclaration(backend structure.BackendDeclaration, timeout time.Duration) (string, error) { - data, err := json.Marshal(backend) - if err != nil { - return "", err - } - ctx, cancel := context.WithTimeout(c.globalCtx, timeout) - defer cancel() - response, err := c.client.EmitWithAck(ctx, utils.ModuleReady, data) - return string(response), err -} - -func (c *SocketLeaderClient) Dial(timeout time.Duration) error { - backOff := backoff.NewExponentialBackOff() - backOff.MaxElapsedTime = timeout - bf := backoff.WithContext(backOff, c.globalCtx) - dial := func() error { - return c.client.Dial(c.globalCtx, c.url) - } - return backoff.Retry(dial, bf) -} - -func (c *SocketLeaderClient) Close() { - c.cancel() - err := c.client.Close() - if err != nil { - log.Warnf(codes.LeaderClientConnectionError, "leader client %s close err: %v", c.url, err) - } - log.Debugf(0, "close previous leader ws connection %s", c.url) -} - -func NewSocketLeaderClient(address string, leaderDisconnectionCallback func()) *SocketLeaderClient { - etpConfig := etp.Config{ - HttpClient: &http.Client{}, - ConnectionReadLimit: WsConnectionReadLimit, - WorkersNum: 1, - WorkersBufferMultiplier: 1, - } - client := etp.NewClient(etpConfig) - leaderClient := &SocketLeaderClient{ - client: client, - url: getURL(address), - } - ctx, cancel := context.WithCancel(context.Background()) - leaderClient.globalCtx = ctx - leaderClient.cancel = cancel - - leaderClient.client.OnDisconnect(func(err error) { - log.WithMetadata(map[string]interface{}{ - "leaderAddr": address, - "error": err, - }).Warn(codes.LeaderClientDisconnected, "disconnected ws from leader") - leaderDisconnectionCallback() - }) - - leaderClient.client.OnError(func(err error) { - log.WithMetadata(map[string]interface{}{ - "leaderAddr": address, - "error": err, - }).Warn(codes.LeaderClientDisconnected, "leader client on error") - }) - leaderClient.client.OnConnect(func() { - log.WithMetadata(map[string]interface{}{ - "leaderAddr": address, - }).Info(codes.LeaderClientConnected, "connected to new leader") - }) - return leaderClient -} - -func getURL(address string) string { - addr, err := net.ResolveTCPAddr("tcp", address) - if err != nil { - panic(err) // must never occurred - } - cfg := config.Get().(*conf.Configuration) - - params := url.Values{} - params.Add(GETParam, "true") - params.Add(utils.ModuleNameGetParamKey, cfg.ModuleName) - return fmt.Sprintf("ws://%s:%d/isp-etp/?%s", addr.IP.String(), addr.Port, params.Encode()) -} diff --git a/codes/codes.go b/codes/codes.go deleted file mode 100644 index ac14f90..0000000 --- a/codes/codes.go +++ /dev/null @@ -1,42 +0,0 @@ -package codes - -const ( - // Common - StartHttpServerError = 1001 - ShutdownHttpServerError = 1002 - ShutdownHttpServerInfo = 1003 - - WebsocketError = 1004 - WebsocketEmitError = 1005 - - DatabaseOperationError = 1006 - - // Raft - InitRaftError = 1007 - RestoreFromRepositoryError = 1008 - RaftLoggerCode = 1009 - BootstrapClusterError = 1010 - RaftShutdownError = 1011 - RaftShutdownInfo = 1012 - - // Raft Log - ApplyLogCommandError = 1013 - PrepareLogCommandError = 1014 - SyncApplyError = 1015 - - // Raft Cluster - SendDeclarationToLeader = 1016 - LeaderClientConnectionError = 1017 - LeaderClientDisconnected = 1018 - LeaderClientConnected = 1023 - LeaderStateChanged = 1024 - LeaderManualDeleteLeader = 1025 - - // Services - DiscoveryServiceSendModulesError = 1019 - RoutesServiceSendRoutesError = 1020 - ConfigServiceBroadcastConfigError = 1021 - - // Mux - InitMuxError = 1022 -) diff --git a/conf/config.yml b/conf/config.yml index 1cd47f9..1867d07 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -1,41 +1,21 @@ -database: - address: "127.0.0.1" - schema: "config_service" - database: "test" - port: "5432" - username: "test" - password: "password" - createSchema: true - -moduleName: "config" +configServiceAddress: + ip: 127.0.0.1 + port: 9001 grpcOuterAddress: - ip: "" - port: 9002 - -ws: - rest: - port: "9001" - ip: "0.0.0.0" - grpc: - port: "9002" - ip: "0.0.0.0" + ip: 127.0.0.1 + port: 9001 +grpcInnerAddress: + ip: 0.0.0.0 + port: 9001 +moduleName: isp-config-service -metrics: - gc: true - memory: true - address: - addressConfiguration: - ip: "0.0.0.0" - port: "9551" - path: "/metrics" +infraServerPort: 9006 -cluster: - dataDir: ./data - inMemory: true - outerAddress: 127.0.0.1:9001 - connectTimeoutSeconds: 5 - bootstrapCluster: true - peers: - - 127.0.0.1:9001 +rqlite: + DataPath: "data" + NodeID: 1 + BootstrapExpect: 1 + RaftAddr: "127.0.0.1:9003" + JoinAddrs: "127.0.0.1:9003" + FKConstraints: true -versionConfigCount: 15 diff --git a/conf/config_test.yml b/conf/config_test.yml deleted file mode 100644 index 7c02876..0000000 --- a/conf/config_test.yml +++ /dev/null @@ -1,7 +0,0 @@ -moduleName: config - -registry: - host: nexus.msp.mobi - -images: - module: nexus.msp.mobi/msp2/dev/isp-config-service:latest diff --git a/conf/configuration.go b/conf/configuration.go deleted file mode 100644 index b8aa2f1..0000000 --- a/conf/configuration.go +++ /dev/null @@ -1,31 +0,0 @@ -//nolint:lll -package conf - -import ( - "github.com/integration-system/isp-lib/v2/structure" -) - -type Configuration struct { - Database structure.DBConfiguration `valid:"required~Required" schema:"База данных,настройка параметров подключения к базе данных"` - GrpcOuterAddress structure.AddressConfiguration `valid:"required~Required" json:"grpcOuterAddress"` - ModuleName string `valid:"required~Required"` - WS WebService `valid:"required~Required" schema:"Конфигурация веб сервиса"` - Metrics structure.MetricConfiguration - Cluster ClusterConfiguration `valid:"required~Required"` - VersionConfigCount int `schema:"Количество сохраняемых старых версий конфигурации"` -} - -type ClusterConfiguration struct { - InMemory bool `schema:"Хранить логи и снэпшоты только в памяти"` - BootstrapCluster bool `schema:"Поднимать кластер и объявлять лидером"` - DataDir string `valid:"required~Required" schema:"Пусть до директории для логов и снэпшотов рафта"` - Peers []string `valid:"required~Required" schema:"Адреса всех нод в кластере,формат address:port"` - OuterAddress string `valid:"required~Required" schema:"Внешний адрес ноды"` - ConnectTimeoutSeconds int `valid:"required~Required" schema:"Таймаут подключения"` -} - -type WebService struct { - Rest structure.AddressConfiguration `valid:"required~Required"` - Grpc structure.AddressConfiguration `valid:"required~Required"` - WsConnectionReadLimitKB int64 `schema:"Максимальное количество килобайт на чтение по вебсокету,при превышении соединение закрывается с ошибкой"` -} diff --git a/conf/default_remote_config.json b/conf/default_remote_config.json similarity index 100% rename from conf/default_remote_config.json rename to conf/default_remote_config.json diff --git a/conf/local.go b/conf/local.go new file mode 100644 index 0000000..e73d137 --- /dev/null +++ b/conf/local.go @@ -0,0 +1,9 @@ +package conf + +import ( + "github.com/txix-open/isp-kit/bootstrap" +) + +type Local struct { + ConfigServiceAddress bootstrap.ConfigServiceAddr +} diff --git a/conf/remote.go b/conf/remote.go new file mode 100644 index 0000000..9f6ec2e --- /dev/null +++ b/conf/remote.go @@ -0,0 +1,4 @@ +package conf + +type Remote struct { +} diff --git a/controller/common_config.go b/controller/common_config.go deleted file mode 100644 index f0ba782..0000000 --- a/controller/common_config.go +++ /dev/null @@ -1,123 +0,0 @@ -package controller - -import ( - "isp-config-service/cluster" - "isp-config-service/domain" - "isp-config-service/entity" - "isp-config-service/service" - "isp-config-service/store" - "isp-config-service/store/state" -) - -var CommonConfig *commonConfig - -type commonConfig struct { - rstore *store.Store -} - -// @Summary Метод получения объектов конфигурации по идентификаторам -// @Description Возвращает массив конфигураций по запрошенным идентификаторам (все, если массив пустой) -// @Tags Общие конфигурации -// @Accept json -// @Produce json -// @Success 200 {array} entity.CommonConfig -// @Router /common_config/get_configs [POST] -func (c *commonConfig) GetConfigs(identities []string) []entity.CommonConfig { - var response []entity.CommonConfig - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - if len(identities) > 0 { - response = state.CommonConfigs().GetByIds(identities) - } else { - response = state.CommonConfigs().GetAll() - } - }) - return response -} - -// @Summary Метод обновления общей конфигурации -// @Description Если конфиг с таким id существует, то обновляет данные, если нет, то добавляет данные в базу -// @Description В случае обновления рассылает всем подключенным модулям актуальную конфигурацию -// @Tags Общие конфигурации -// @Accept json -// @Produce json -// @Param body body entity.CommonConfig true "объект для сохранения" -// @Success 200 {object} entity.CommonConfig -// @Failure 404 {object} structure.GrpcError "если конфигурация не найдена" -// @Failure 500 {object} structure.GrpcError -// @Router /common_config/create_update_config [POST] -func (c *commonConfig) CreateUpdateConfig(config entity.CommonConfig) (*entity.CommonConfig, error) { - var response entity.CommonConfig - now := state.GenerateDate() - config.CreatedAt = now - config.UpdatedAt = now - upsertConfig := cluster.UpsertCommonConfig{ - Config: config, - } - if config.Id == "" { - upsertConfig.Config.Id = state.GenerateId() - upsertConfig.Create = true - } - command := cluster.PrepareUpsertCommonConfigCommand(upsertConfig) - err := PerformSyncApplyWithError(command, &response) - if err != nil { - return nil, err - } - return &response, nil -} - -// @Summary Метод удаления объектов общей конфигурации по идентификаторам -// @Description Возвращает флаг удаления и набор связей с модулями и конфигурациями, в случае наличия связей deleted всегда false -// @Tags Общие конфигурации -// @Accept json -// @Produce json -// @Param body body domain.ConfigIdRequest true "id общей конфигурации" -// @Success 200 {object} domain.DeleteCommonConfigResponse -// @Failure 500 {object} structure.GrpcError -// @Router /common_config/delete_config [POST] -func (c *commonConfig) DeleteConfigs(req domain.ConfigIdRequest) (*domain.DeleteCommonConfigResponse, error) { - var response domain.DeleteCommonConfigResponse - command := cluster.PrepareDeleteCommonConfigsCommand(req.Id) - err := PerformSyncApplyWithError(command, &response) - if err != nil { - return nil, err - } - return &response, nil -} - -// @Summary Метод компиляции итоговой конфигурации для модулей -// @Description Возвращает скомпилированный объект конфигурации -// @Tags Общие конфигурации -// @Accept json -// @Produce json -// @Param body body domain.CompileConfigsRequest true "перечисление идентификаторов общей конфигурации и исхдных конфиг" -// @Success 200 {object} domain.CompiledConfigResponse -// @Router /common_config/compile [POST] -func (c *commonConfig) CompileConfigs(req domain.CompileConfigsRequest) domain.CompiledConfigResponse { - var result map[string]interface{} - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - result = service.ConfigService.CompileConfig(req.Data, state, req.CommonConfigsIdList...) - }) - return result -} - -// @Summary Метод получения связей общей конфигурациями с конфигурацией модулей -// @Description Возвращает ассоциативный массив, ключами которого являются название модулей, а значения - название конфигурации модуля -// @Tags Общие конфигурации -// @Accept json -// @Produce json -// @Param body body domain.ConfigIdRequest true "id общей конфигурации" -// @Success 200 {object} domain.CommonConfigLinks -// @Router /common_config/get_links [POST] -func (c *commonConfig) GetLinks(req domain.ConfigIdRequest) domain.CommonConfigLinks { - var result domain.CommonConfigLinks - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - result = service.CommonConfig.GetCommonConfigLinks(req.Id, state) - }) - return result -} - -func NewCommonConfig(rstore *store.Store) *commonConfig { - return &commonConfig{ - rstore: rstore, - } -} diff --git a/controller/config.go b/controller/config.go deleted file mode 100644 index bc2c8b2..0000000 --- a/controller/config.go +++ /dev/null @@ -1,236 +0,0 @@ -package controller - -import ( - "time" - - "github.com/integration-system/isp-lib/v2/utils" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "isp-config-service/cluster" - "isp-config-service/domain" - "isp-config-service/entity" - "isp-config-service/service" - "isp-config-service/store" - "isp-config-service/store/state" -) - -var Config *config - -type config struct { - rstore *store.Store -} - -// @Summary Метод получения объекта конфигурации по названию модуля -// @Description Возвращает активную конфиграцию по названию модуля -// @Tags Конфигурация -// @Accept json -// @Produce json -// @Param body body domain.GetByModuleNameRequest true "название модуля" -// @Success 200 {object} entity.Config -// @Failure 404 {object} structure.GrpcError "если конфигурация не найдена" -// @Failure 500 {object} structure.GrpcError -// @Router /config/get_active_config_by_module_name [POST] -func (c *config) GetActiveConfigByModuleName(request domain.GetByModuleNameRequest) (*entity.Config, error) { - var ( - module *entity.Module - config *entity.Config - ) - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - module = state.Modules().GetByName(request.ModuleName) - if module != nil { - config = state.Configs().GetActiveByModuleId(module.Id) - } - }) - if module == nil { - return nil, status.Errorf(codes.NotFound, "module with name '%s' not found", request.ModuleName) - } - if config == nil { - return nil, status.Errorf(codes.NotFound, "active config for module '%s' not found", request.ModuleName) - } - return config, nil -} - -// @Summary Метод получения списка конфигураций по ID модуля -// @Description Возвращает список конфиграции по ID модуля -// @Tags Конфигурация -// @Accept json -// @Produce json -// @Param body body domain.GetByModuleIdRequest true "ID модуля" -// @Success 200 {array} domain.ConfigModuleInfo -// @Failure 400 {object} structure.GrpcError "если идентификатор не указан" -// @Failure 404 {object} structure.GrpcError "если конфигурация не найдена" -// @Failure 500 {object} structure.GrpcError -// @Router /config/get_configs_by_module_id [POST] -func (c *config) GetConfigsByModuleId(request domain.GetByModuleIdRequest) ([]domain.ConfigModuleInfo, error) { - var config []entity.Config - var configInfo []domain.ConfigModuleInfo - - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - config = state.Configs().GetByModuleIds([]string{request.ModuleId}) - if len(config) == 0 { - return - } - for _, value := range config { - configInfo = append(configInfo, domain.ConfigModuleInfo{ - Config: value, - Valid: service.ModuleRegistry.ValidateConfig(value, state), - }) - } - }) - - if len(configInfo) == 0 { - return nil, status.Errorf(codes.NotFound, "configs for module '%s' not found", request.ModuleId) - } - - return configInfo, nil -} - -// @Summary Метод обновления конфигурации -// @Description Если конфиг с таким id существует, то обновляет данные, если нет, то добавляет данные в базу -// @Description В случае обновления рассылает всем подключенным модулям актуальную конфигурацию -// @Tags Конфигурация -// @Accept json -// @Produce json -// @Param body body domain.CreateUpdateConfigRequest true "объект для сохранения" -// @Success 200 {object} domain.ConfigModuleInfo -// @Failure 404 {object} structure.GrpcError "если конфигурация не найдена" -// @Failure 500 {object} structure.GrpcError -// @Router /config/create_update_config [POST] -func (c *config) CreateUpdateConfig(config domain.CreateUpdateConfigRequest) (*domain.ConfigModuleInfo, error) { - var response domain.CreateUpdateConfigResponse - now := state.GenerateDate() - config.CreatedAt = now - config.UpdatedAt = now - upsertConfig := cluster.UpsertConfig{ - Config: config.Config, - Unsafe: config.Unsafe, - VersionId: state.GenerateId(), - VersionCreateAt: time.Now(), - } - if config.Id == "" { - upsertConfig.Config.Id = state.GenerateId() - upsertConfig.Create = true - } - command := cluster.PrepareUpsertConfigCommand(upsertConfig) - err := PerformSyncApplyWithError(command, &response) - if err != nil { - return nil, err - } else if response.ErrorDetails != nil { - return nil, utils.CreateValidationErrorDetails(codes.InvalidArgument, utils.ValidationError, response.ErrorDetails) - } - - return response.Config, nil -} - -// @Summary Метод активации конфигурации для модуля -// @Description Активирует указанную конфигурацию и деактивирует остальные, возвращает активированную конфигурацию -// @Tags Конфигурация -// @Accept json -// @Produce json -// @Param body body domain.ConfigIdRequest true "id конфигурации для изменения" -// @Success 200 {object} entity.Config "активированная конфигурация" -// @Failure 404 {object} structure.GrpcError "если конфигурация не найдена" -// @Failure 500 {object} structure.GrpcError -// @Router /config/mark_config_as_active [POST] -func (c *config) MarkConfigAsActive(identity domain.ConfigIdRequest) (*entity.Config, error) { - var response entity.Config - command := cluster.PrepareActivateConfigCommand(identity.Id, state.GenerateDate()) - err := PerformSyncApplyWithError(command, &response) - if err != nil { - return nil, err - } - return &response, nil -} - -// @Summary Метод удаления объектов конфигурации по идентификаторам -// @Description Возвращает количество удаленных модулей -// @Tags Конфигурация -// @Accept json -// @Produce json -// @Param body body []string true "массив идентификаторов конфигураций" -// @Success 200 {object} domain.DeleteResponse -// @Failure 400 {object} structure.GrpcError "если не указан массив идентификаторов" -// @Failure 500 {object} structure.GrpcError -// @Router /config/delete_config [POST] -func (c *config) DeleteConfigs(identities []string) (*domain.DeleteResponse, error) { - if len(identities) == 0 { - validationErrors := map[string]string{ - "ids": "Required", - } - return nil, utils.CreateValidationErrorDetails(codes.InvalidArgument, utils.ValidationError, validationErrors) - } - var response domain.DeleteResponse - command := cluster.PrepareDeleteConfigsCommand(identities) - err := PerformSyncApply(command, &response) - if err != nil { - return nil, err - } - return &response, nil -} - -// @Summary Метод удаления версии конфигурации -// @Description Возвращает количество удаленных версий -// @Tags Конфигурация -// @Accept json -// @Produce json -// @Param body body domain.ConfigIdRequest true "id версии конфигурации" -// @Success 200 {object} domain.DeleteResponse -// @Failure 400 {object} structure.GrpcError "если не указан массив идентификаторов" -// @Failure 500 {object} structure.GrpcError -// @Router /config/delete_version [POST] -func (c *config) DeleteConfigVersion(req domain.ConfigIdRequest) (*domain.DeleteResponse, error) { - var response domain.DeleteResponse - command := cluster.PrepareDeleteConfigVersionCommand(req.Id) - err := PerformSyncApplyWithError(command, &response) - if err != nil { - return nil, err - } - return &response, nil -} - -// @Summary Метод получение старых версий конфигурации -// @Description Возвращает предыдущие версии конфигураций -// @Tags Конфигурация -// @Accept json -// @Produce json -// @Param body body domain.ConfigIdRequest true "id конфигурации" -// @Success 200 {array} entity.VersionConfig -// @Failure 400 {object} structure.GrpcError "если не указан массив идентификаторов" -// @Failure 500 {object} structure.GrpcError -// @Router /config/get_all_version [POST] -func (c *config) GetAllVersion(req domain.ConfigIdRequest) ([]entity.VersionConfig, error) { - var response []entity.VersionConfig - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - response = state.VersionConfig().GetByConfigId(req.Id) - }) - return response, nil -} - -// @Summary Метод получение актуальной конфигурации конфигурации -// @Description Возвращает актуальную версию конфигурации без дополнительного содержимого (ConfigData) -// @Tags Конфигурация -// @Accept json -// @Produce json -// @Param body body domain.ConfigIdRequest true "id конфигурации" -// @Success 200 {object} entity.Config -// @Failure 400 {object} structure.GrpcError "если не указан идентификатор конфигурации" -// @Failure 500 {object} structure.GrpcError -// @Router /config/get_config_by_id [POST] -func (c *config) GetConfigById(req domain.ConfigIdRequest) (entity.Config, error) { - var response []entity.Config - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - response = state.Configs().GetByIds([]string{req.Id}) - }) - - if len(response) > 0 { - return response[0], nil - } - - return entity.Config{}, status.Errorf(codes.NotFound, "config with id '%s' not found", req.Id) -} - -func NewConfig(rstore *store.Store) *config { - return &config{ - rstore: rstore, - } -} diff --git a/controller/helpers.go b/controller/helpers.go deleted file mode 100644 index 32f851a..0000000 --- a/controller/helpers.go +++ /dev/null @@ -1,72 +0,0 @@ -package controller - -import ( - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - jsoniter "github.com/json-iterator/go" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "isp-config-service/cluster" - codes2 "isp-config-service/codes" - "isp-config-service/holder" -) - -var json = jsoniter.ConfigFastest - -func unknownError() error { - return status.Error(codes.Unknown, utils.ServiceError) -} - -func PerformSyncApply(command []byte, responsePtr interface{}) error { - applyLogResponse, err := holder.ClusterClient.SyncApply(command) - if err != nil { - cmd := cluster.ParseCommand(command) - log.Warnf(codes2.SyncApplyError, "apply %s: %v", cmd, err) - return unknownError() - } - if applyLogResponse != nil && applyLogResponse.ApplyError != "" { - commandName := cluster.ParseCommand(command).String() - log.WithMetadata(map[string]interface{}{ - "result": string(applyLogResponse.Result), - "applyError": applyLogResponse.ApplyError, - "commandName": commandName, - }).Warnf(codes2.SyncApplyError, "apply command") - return unknownError() - } - if responsePtr != nil { - err = json.Unmarshal(applyLogResponse.Result, responsePtr) - if err != nil { - return unknownError() - } - } - return nil -} - -func PerformSyncApplyWithError(command []byte, responsePtr interface{}) error { - applyLogResponse, err := holder.ClusterClient.SyncApply(command) - if err != nil { - cmd := cluster.ParseCommand(command) - log.Warnf(codes2.SyncApplyError, "apply %s: %v", cmd, err) - return unknownError() - } - if applyLogResponse != nil && applyLogResponse.ApplyError != "" { - commandName := cluster.ParseCommand(command).String() - log.WithMetadata(map[string]interface{}{ - "result": string(applyLogResponse.Result), - "applyError": applyLogResponse.ApplyError, - "commandName": commandName, - }).Warnf(codes2.SyncApplyError, "apply command") - return unknownError() - } - clusterResp := cluster.ResponseWithError{ - Response: responsePtr, - } - err = json.Unmarshal(applyLogResponse.Result, &clusterResp) - if err != nil { - return unknownError() - } - if clusterResp.Error != "" { - return status.Error(clusterResp.ErrorCode, clusterResp.Error) - } - return nil -} diff --git a/controller/module.go b/controller/module.go index c888d63..146b5e8 100644 --- a/controller/module.go +++ b/controller/module.go @@ -1,117 +1,39 @@ package controller import ( - "google.golang.org/grpc/status" - "isp-config-service/cluster" - "isp-config-service/domain" - "isp-config-service/entity" - "isp-config-service/service" - "isp-config-service/store" - "isp-config-service/store/state" - "time" + "context" - "github.com/integration-system/isp-lib/v2/utils" - "google.golang.org/grpc/codes" + "github.com/txix-open/etp/v3" + "github.com/txix-open/etp/v3/msg" ) -const ( - defaultEventLifetime = 5 * time.Second -) - -var Module *module +type Module struct { +} -type module struct { - rstore *store.Store +func NewModule() Module { + return Module{} } -// @Summary Метод получения полной информации о состоянии модуля -// @Description Возвращает полное состояние всех модулей в кластере (конфигурация, схема конфигурации, подключенные экземпляры) -// @Tags Модули -// @Accept json -// @Produce json -// @Success 200 {array} domain.ModuleInfo -// @Failure 500 {object} structure.GrpcError -// @Router /module/get_modules_info [POST] -func (c *module) GetModulesAggregatedInfo() ([]domain.ModuleInfo, error) { - var response []domain.ModuleInfo - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - response = service.ModuleRegistry.GetAggregatedModuleInfo(state) - }) - return response, nil +func (m Module) OnConnect(conn *etp.Conn) { + } -// @Summary Метод удаления объектов модулей по идентификаторам -// @Description Возвращает количество удаленных модулей -// @Tags Модули -// @Accept json -// @Produce json -// @Param body body []string true "массив идентификаторов модулей" -// @Success 200 {object} domain.DeleteResponse -// @Failure 500 {object} structure.GrpcError -// @Router /module/delete_module [POST] -func (*module) DeleteModules(identities []string) (*domain.DeleteResponse, error) { - if len(identities) == 0 { - validationErrors := map[string]string{ - "ids": "Required", - } - return nil, utils.CreateValidationErrorDetails(codes.InvalidArgument, utils.ValidationError, validationErrors) - } +func (m Module) OnDisconnect(ctx *etp.Conn, conn error) { - var deleteResponse domain.DeleteResponse - command := cluster.PrepareDeleteModulesCommand(identities) - err := PerformSyncApply(command, &deleteResponse) - if err != nil { - return nil, err - } - return &deleteResponse, err } -// @Summary Метод получения модуля по имени -// @Tags Модули -// @Accept json -// @Produce json -// @Param body body domain.GetByModuleNameRequest true "название модуля" -// @Success 200 {object} entity.Module -// @Failure 500 {object} structure.GrpcError -// @Failure 404 {object} structure.GrpcError "модуль с указанным названием не найден" -// @Router /module/get_by_name [POST] -func (c *module) GetModuleByName(req domain.GetByModuleNameRequest) (*entity.Module, error) { - var module *entity.Module - c.rstore.VisitReadonlyState(func(readonlyState state.ReadonlyState) { - module = readonlyState.Modules().GetByName(req.ModuleName) - }) - if module == nil { - return nil, status.Errorf(codes.NotFound, "module with name '%s' not found", req.ModuleName) - } - return module, nil +func (m Module) OnError(conn *etp.Conn, err error) { + } -// @Summary Метод отправки события всем подключенным модулям -// @Tags Модули -// @Accept json -// @Produce json -// @Param body body domain.BroadcastEventRequest true "событие" -// @Success 200 "OK" -// @Failure 500 {object} structure.GrpcError -// @Router /module/broadcast_event [POST] -func (c *module) BroadcastEvent(req domain.BroadcastEventRequest) error { - cmd := cluster.BroadcastEvent{ - Event: req.Event, - ModuleNames: req.ModuleNames, - Payload: req.Payload, - PerformUntil: time.Now().UTC().Add(defaultEventLifetime), - } - command := cluster.PrepareBroadcastEventCommand(cmd) - err := PerformSyncApply(command, nil) - if err != nil { - return err - } +func (m Module) OnModuleReady(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { + return nil +} +func (m Module) OnModuleRequirements(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { return nil } -func NewModule(rstore *store.Store) *module { - return &module{ - rstore: rstore, - } +func (m Module) OnModuleConfigSchema(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { + return nil } diff --git a/controller/route.go b/controller/route.go deleted file mode 100644 index 3494245..0000000 --- a/controller/route.go +++ /dev/null @@ -1,36 +0,0 @@ -package controller - -import ( - "github.com/integration-system/isp-lib/v2/structure" - "isp-config-service/store" - "isp-config-service/store/state" -) - -var ( - Routes *routes -) - -type routes struct { - rstore *store.Store -} - -// @Summary Метод получения маршрутов -// @Description Возвращает все доступные модули -// @Tags Роуты -// @Accept json -// @Produce json -// @Success 200 {array} structure.BackendDeclaration -// @Router /routing/get_routes [POST] -func (c *routes) GetRoutes() ([]structure.BackendDeclaration, error) { - var response structure.RoutingConfig - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - response = state.Mesh().GetRoutes() - }) - return response, nil -} - -func NewRoutes(rstore *store.Store) *routes { - return &routes{ - rstore: rstore, - } -} diff --git a/controller/schema.go b/controller/schema.go deleted file mode 100644 index 743a0b6..0000000 --- a/controller/schema.go +++ /dev/null @@ -1,43 +0,0 @@ -package controller - -import ( - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "isp-config-service/domain" - "isp-config-service/entity" - "isp-config-service/store" - "isp-config-service/store/state" -) - -var Schema *schema - -type schema struct { - rstore *store.Store -} - -// @Summary Метод получения схемы конфигурации модуля -// @Description Возвращает текущую json схему конфигурации модуля -// @Tags Схема -// @Accept json -// @Produce json -// @Param body body domain.GetByModuleIdRequest true "идентификатор модуля" -// @Success 200 {object} entity.ConfigSchema -// @Failure 404 {object} structure.GrpcError "если схема для модуля не найдена" -// @Failure 500 {object} structure.GrpcError -// @Router /schema/get_by_module_id [POST] -func (c *schema) GetByModuleId(request domain.GetByModuleIdRequest) (*entity.ConfigSchema, error) { - var result []entity.ConfigSchema - c.rstore.VisitReadonlyState(func(state state.ReadonlyState) { - result = state.Schemas().GetByModuleIds([]string{request.ModuleId}) - }) - if len(result) == 0 { - return nil, status.Errorf(codes.NotFound, "schema with moduleId %s not found", request.ModuleId) - } - return &result[0], nil -} - -func NewSchema(rstore *store.Store) *schema { - return &schema{ - rstore: rstore, - } -} diff --git a/docs/swagger.yaml b/docs/swagger.yaml deleted file mode 100644 index 37996bf..0000000 --- a/docs/swagger.yaml +++ /dev/null @@ -1,1102 +0,0 @@ -basePath: /api/config -definitions: - domain.BroadcastEventRequest: - properties: - event: - type: string - moduleNames: - items: - type: string - type: array - payload: - items: - type: integer - type: array - type: object - domain.CommonConfigLinks: - additionalProperties: - items: - type: string - type: array - type: object - domain.CompileConfigsRequest: - properties: - commonConfigsIdList: - items: - type: string - type: array - data: - additionalProperties: true - type: object - type: object - domain.CompiledConfigResponse: - additionalProperties: true - type: object - domain.ConfigIdRequest: - properties: - id: - type: string - type: object - domain.ConfigModuleInfo: - properties: - active: - type: boolean - commonConfigs: - items: - type: string - type: array - createdAt: - type: string - data: - $ref: '#/definitions/entity.ConfigData' - description: - type: string - id: - type: string - moduleId: - type: string - name: - type: string - updatedAt: - type: string - valid: - type: boolean - version: - type: integer - type: object - domain.Connection: - properties: - address: - $ref: '#/definitions/structure.AddressConfiguration' - endpoints: - items: - $ref: '#/definitions/structure.EndpointDescriptor' - type: array - establishedAt: - type: string - libVersion: - type: string - version: - type: string - type: object - domain.CreateUpdateConfigRequest: - properties: - active: - type: boolean - commonConfigs: - items: - type: string - type: array - createdAt: - type: string - data: - $ref: '#/definitions/entity.ConfigData' - description: - type: string - id: - type: string - moduleId: - type: string - name: - type: string - unsafe: - type: boolean - updatedAt: - type: string - version: - type: integer - type: object - domain.DeleteCommonConfigResponse: - properties: - deleted: - type: boolean - links: - $ref: '#/definitions/domain.CommonConfigLinks' - type: object - domain.DeleteResponse: - properties: - deleted: - type: integer - type: object - domain.GetByModuleIdRequest: - properties: - moduleId: - type: string - type: object - domain.GetByModuleNameRequest: - properties: - moduleName: - type: string - type: object - domain.ModuleDependency: - properties: - id: - type: string - name: - type: string - required: - type: boolean - type: object - domain.ModuleInfo: - properties: - active: - type: boolean - configSchema: - $ref: '#/definitions/schema.Schema' - configs: - items: - $ref: '#/definitions/domain.ConfigModuleInfo' - type: array - createdAt: - type: string - id: - type: string - lastConnectedAt: - type: string - lastDisconnectedAt: - type: string - name: - type: string - requiredModules: - items: - $ref: '#/definitions/domain.ModuleDependency' - type: array - status: - items: - $ref: '#/definitions/domain.Connection' - type: array - type: object - entity.CommonConfig: - properties: - createdAt: - type: string - data: - $ref: '#/definitions/entity.ConfigData' - description: - type: string - id: - type: string - name: - type: string - updatedAt: - type: string - type: object - entity.Config: - properties: - active: - type: boolean - commonConfigs: - items: - type: string - type: array - createdAt: - type: string - data: - $ref: '#/definitions/entity.ConfigData' - description: - type: string - id: - type: string - moduleId: - type: string - name: - type: string - updatedAt: - type: string - version: - type: integer - type: object - entity.ConfigData: - additionalProperties: true - type: object - entity.ConfigSchema: - properties: - createdAt: - type: string - id: - type: string - moduleId: - type: string - schema: - $ref: '#/definitions/schema.Schema' - updatedAt: - type: string - version: - type: string - type: object - entity.Module: - properties: - createdAt: - type: string - id: - type: string - lastConnectedAt: - type: string - lastDisconnectedAt: - type: string - name: - type: string - type: object - entity.VersionConfig: - properties: - configId: - type: string - configVersion: - type: integer - createdAt: - type: string - data: - $ref: '#/definitions/entity.ConfigData' - id: - type: string - type: object - jsonschema.Definitions: - additionalProperties: - $ref: '#/definitions/jsonschema.Type' - type: object - jsonschema.Type: - properties: - $ref: - description: section 7 - type: string - $schema: - description: RFC draft-wright-json-schema-00 - type: string - additionalItems: - allOf: - - $ref: '#/definitions/jsonschema.Type' - description: section 5.9 - additionalProperties: - description: section 5.18 - items: - type: integer - type: array - allOf: - description: section 5.22 - items: - $ref: '#/definitions/jsonschema.Type' - type: array - anyOf: - description: section 5.23 - items: - $ref: '#/definitions/jsonschema.Type' - type: array - binaryEncoding: - description: section 4.3 - type: string - default: - description: section 6.2 - definitions: - allOf: - - $ref: '#/definitions/jsonschema.Definitions' - description: section 5.26 - dependencies: - additionalProperties: - $ref: '#/definitions/jsonschema.Type' - description: section 5.19 - type: object - description: - description: section 6.1 - type: string - enum: - description: section 5.20 - items: {} - type: array - exclusiveMaximum: - description: section 5.3 - type: boolean - exclusiveMinimum: - description: section 5.5 - type: boolean - format: - description: section 7 - type: string - items: - allOf: - - $ref: '#/definitions/jsonschema.Type' - description: section 5.9 - maxItems: - description: section 5.10 - type: integer - maxLength: - description: section 5.6 - type: integer - maxProperties: - description: section 5.13 - type: integer - maximum: - description: section 5.2 - type: integer - media: - allOf: - - $ref: '#/definitions/jsonschema.Type' - description: RFC draft-wright-json-schema-hyperschema-00, section 4 - minItems: - description: section 5.11 - type: integer - minLength: - description: section 5.7 - type: integer - minProperties: - description: section 5.14 - type: integer - minimum: - description: section 5.4 - type: integer - multipleOf: - description: RFC draft-wright-json-schema-validation-00, section 5 - type: integer - not: - allOf: - - $ref: '#/definitions/jsonschema.Type' - description: section 5.25 - oneOf: - description: section 5.24 - items: - $ref: '#/definitions/jsonschema.Type' - type: array - pattern: - description: section 5.8 - type: string - patternProperties: - additionalProperties: - $ref: '#/definitions/jsonschema.Type' - description: section 5.17 - type: object - properties: - additionalProperties: - $ref: '#/definitions/jsonschema.Type' - description: section 5.16 - type: object - required: - description: section 5.15 - items: - type: string - type: array - title: - description: RFC draft-wright-json-schema-validation-00, section 6, 7 - type: string - type: - description: section 5.21 - type: string - uniqueItems: - description: section 5.12 - type: boolean - type: object - schema.Schema: - properties: - $ref: - description: section 7 - type: string - $schema: - description: RFC draft-wright-json-schema-00 - type: string - additionalItems: - allOf: - - $ref: '#/definitions/jsonschema.Type' - description: section 5.9 - additionalProperties: - description: section 5.18 - items: - type: integer - type: array - allOf: - description: section 5.22 - items: - $ref: '#/definitions/jsonschema.Type' - type: array - anyOf: - description: section 5.23 - items: - $ref: '#/definitions/jsonschema.Type' - type: array - binaryEncoding: - description: section 4.3 - type: string - default: - description: section 6.2 - definitions: - $ref: '#/definitions/jsonschema.Definitions' - dependencies: - additionalProperties: - $ref: '#/definitions/jsonschema.Type' - description: section 5.19 - type: object - description: - description: section 6.1 - type: string - enum: - description: section 5.20 - items: {} - type: array - exclusiveMaximum: - description: section 5.3 - type: boolean - exclusiveMinimum: - description: section 5.5 - type: boolean - format: - description: section 7 - type: string - items: - allOf: - - $ref: '#/definitions/jsonschema.Type' - description: section 5.9 - maxItems: - description: section 5.10 - type: integer - maxLength: - description: section 5.6 - type: integer - maxProperties: - description: section 5.13 - type: integer - maximum: - description: section 5.2 - type: integer - media: - allOf: - - $ref: '#/definitions/jsonschema.Type' - description: RFC draft-wright-json-schema-hyperschema-00, section 4 - minItems: - description: section 5.11 - type: integer - minLength: - description: section 5.7 - type: integer - minProperties: - description: section 5.14 - type: integer - minimum: - description: section 5.4 - type: integer - multipleOf: - description: RFC draft-wright-json-schema-validation-00, section 5 - type: integer - not: - allOf: - - $ref: '#/definitions/jsonschema.Type' - description: section 5.25 - oneOf: - description: section 5.24 - items: - $ref: '#/definitions/jsonschema.Type' - type: array - pattern: - description: section 5.8 - type: string - patternProperties: - additionalProperties: - $ref: '#/definitions/jsonschema.Type' - description: section 5.17 - type: object - properties: - additionalProperties: - $ref: '#/definitions/jsonschema.Type' - description: section 5.16 - type: object - required: - description: section 5.15 - items: - type: string - type: array - title: - description: RFC draft-wright-json-schema-validation-00, section 6, 7 - type: string - type: - description: section 5.21 - type: string - uniqueItems: - description: section 5.12 - type: boolean - type: object - structure.AddressConfiguration: - properties: - ip: - type: string - port: - type: string - type: object - structure.BackendDeclaration: - properties: - address: - $ref: '#/definitions/structure.AddressConfiguration' - endpoints: - items: - $ref: '#/definitions/structure.EndpointDescriptor' - type: array - libVersion: - type: string - moduleName: - type: string - requiredModules: - items: - $ref: '#/definitions/structure.ModuleDependency' - type: array - version: - type: string - type: object - structure.EndpointDescriptor: - properties: - extra: - additionalProperties: true - type: object - inner: - type: boolean - path: - type: string - userAuthRequired: - type: boolean - type: object - structure.GrpcError: - properties: - details: - items: {} - type: array - errorCode: - type: string - errorMessage: - type: string - type: object - structure.ModuleDependency: - properties: - name: - type: string - required: - type: boolean - type: object -host: localhost:9003 -info: - contact: {} - description: Сервис управления конфигурацией модулей ISP кластера - license: - name: GNU GPL v3.0 - title: ISP configuration service - version: 2.4.1 -paths: - /common_config/compile: - post: - consumes: - - application/json - description: Возвращает скомпилированный объект конфигурации - parameters: - - description: перечисление идентификаторов общей конфигурации и исхдных конфиг - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.CompileConfigsRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/domain.CompiledConfigResponse' - summary: Метод компиляции итоговой конфигурации для модулей - tags: - - Общие конфигурации - /common_config/create_update_config: - post: - consumes: - - application/json - description: |- - Если конфиг с таким id существует, то обновляет данные, если нет, то добавляет данные в базу - В случае обновления рассылает всем подключенным модулям актуальную конфигурацию - parameters: - - description: объект для сохранения - in: body - name: body - required: true - schema: - $ref: '#/definitions/entity.CommonConfig' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/entity.CommonConfig' - "404": - description: если конфигурация не найдена - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод обновления общей конфигурации - tags: - - Общие конфигурации - /common_config/delete_config: - post: - consumes: - - application/json - description: Возвращает флаг удаления и набор связей с модулями и конфигурациями, - в случае наличия связей deleted всегда false - parameters: - - description: id общей конфигурации - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.ConfigIdRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/domain.DeleteCommonConfigResponse' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод удаления объектов общей конфигурации по идентификаторам - tags: - - Общие конфигурации - /common_config/get_configs: - post: - consumes: - - application/json - description: Возвращает массив конфигураций по запрошенным идентификаторам (все, - если массив пустой) - produces: - - application/json - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/entity.CommonConfig' - type: array - summary: Метод получения объектов конфигурации по идентификаторам - tags: - - Общие конфигурации - /common_config/get_links: - post: - consumes: - - application/json - description: Возвращает ассоциативный массив, ключами которого являются название - модулей, а значения - название конфигурации модуля - parameters: - - description: id общей конфигурации - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.ConfigIdRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/domain.CommonConfigLinks' - summary: Метод получения связей общей конфигурациями с конфигурацией модулей - tags: - - Общие конфигурации - /config/create_update_config: - post: - consumes: - - application/json - description: |- - Если конфиг с таким id существует, то обновляет данные, если нет, то добавляет данные в базу - В случае обновления рассылает всем подключенным модулям актуальную конфигурацию - parameters: - - description: объект для сохранения - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.CreateUpdateConfigRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/domain.ConfigModuleInfo' - "404": - description: если конфигурация не найдена - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод обновления конфигурации - tags: - - Конфигурация - /config/delete_config: - post: - consumes: - - application/json - description: Возвращает количество удаленных модулей - parameters: - - description: массив идентификаторов конфигураций - in: body - name: body - required: true - schema: - items: - type: string - type: array - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/domain.DeleteResponse' - "400": - description: если не указан массив идентификаторов - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод удаления объектов конфигурации по идентификаторам - tags: - - Конфигурация - /config/delete_version: - post: - consumes: - - application/json - description: Возвращает количество удаленных версий - parameters: - - description: id версии конфигурации - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.ConfigIdRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/domain.DeleteResponse' - "400": - description: если не указан массив идентификаторов - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод удаления версии конфигурации - tags: - - Конфигурация - /config/get_active_config_by_module_name: - post: - consumes: - - application/json - description: Возвращает активную конфиграцию по названию модуля - parameters: - - description: название модуля - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.GetByModuleNameRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/entity.Config' - "404": - description: если конфигурация не найдена - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод получения объекта конфигурации по названию модуля - tags: - - Конфигурация - /config/get_all_version: - post: - consumes: - - application/json - description: Возвращает предыдущие версии конфигураций - parameters: - - description: id конфигурации - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.ConfigIdRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/entity.VersionConfig' - type: array - "400": - description: если не указан массив идентификаторов - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод получение старых версий конфигурации - tags: - - Конфигурация - /config/get_config_by_id: - post: - consumes: - - application/json - description: Возвращает актуальную версию конфигурации без дополнительного содержимого - (ConfigData) - parameters: - - description: id конфигурации - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.ConfigIdRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/entity.Config' - "400": - description: если не указан идентификатор конфигурации - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод получение актуальной конфигурации конфигурации - tags: - - Конфигурация - /config/get_configs_by_module_id: - post: - consumes: - - application/json - description: Возвращает список конфиграции по ID модуля - parameters: - - description: ID модуля - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.GetByModuleIdRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/domain.ConfigModuleInfo' - type: array - "400": - description: если идентификатор не указан - schema: - $ref: '#/definitions/structure.GrpcError' - "404": - description: если конфигурация не найдена - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод получения списка конфигураций по ID модуля - tags: - - Конфигурация - /config/mark_config_as_active: - post: - consumes: - - application/json - description: Активирует указанную конфигурацию и деактивирует остальные, возвращает - активированную конфигурацию - parameters: - - description: id конфигурации для изменения - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.ConfigIdRequest' - produces: - - application/json - responses: - "200": - description: активированная конфигурация - schema: - $ref: '#/definitions/entity.Config' - "404": - description: если конфигурация не найдена - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод активации конфигурации для модуля - tags: - - Конфигурация - /module/broadcast_event: - post: - consumes: - - application/json - parameters: - - description: событие - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.BroadcastEventRequest' - produces: - - application/json - responses: - "200": - description: OK - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод отправки события всем подключенным модулям - tags: - - Модули - /module/delete_module: - post: - consumes: - - application/json - description: Возвращает количество удаленных модулей - parameters: - - description: массив идентификаторов модулей - in: body - name: body - required: true - schema: - items: - type: string - type: array - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/domain.DeleteResponse' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод удаления объектов модулей по идентификаторам - tags: - - Модули - /module/get_by_name: - post: - consumes: - - application/json - parameters: - - description: название модуля - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.GetByModuleNameRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/entity.Module' - "404": - description: модуль с указанным названием не найден - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод получения модуля по имени - tags: - - Модули - /module/get_modules_info: - post: - consumes: - - application/json - description: Возвращает полное состояние всех модулей в кластере (конфигурация, - схема конфигурации, подключенные экземпляры) - produces: - - application/json - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/domain.ModuleInfo' - type: array - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод получения полной информации о состоянии модуля - tags: - - Модули - /routing/get_routes: - post: - consumes: - - application/json - description: Возвращает все доступные модули - produces: - - application/json - responses: - "200": - description: OK - schema: - items: - $ref: '#/definitions/structure.BackendDeclaration' - type: array - summary: Метод получения маршрутов - tags: - - Роуты - /schema/get_by_module_id: - post: - consumes: - - application/json - description: Возвращает текущую json схему конфигурации модуля - parameters: - - description: идентификатор модуля - in: body - name: body - required: true - schema: - $ref: '#/definitions/domain.GetByModuleIdRequest' - produces: - - application/json - responses: - "200": - description: OK - schema: - $ref: '#/definitions/entity.ConfigSchema' - "404": - description: если схема для модуля не найдена - schema: - $ref: '#/definitions/structure.GrpcError' - "500": - description: Internal Server Error - schema: - $ref: '#/definitions/structure.GrpcError' - summary: Метод получения схемы конфигурации модуля - tags: - - Схема -swagger: "2.0" diff --git a/domain/request.go b/domain/request.go deleted file mode 100644 index 13345fe..0000000 --- a/domain/request.go +++ /dev/null @@ -1,35 +0,0 @@ -package domain - -import ( - json2 "encoding/json" - - "isp-config-service/entity" -) - -type ConfigIdRequest struct { - Id string `json:"id" valid:"required~Required"` -} - -type CompileConfigsRequest struct { - Data map[string]interface{} - CommonConfigsIdList []string -} - -type GetByModuleIdRequest struct { - ModuleId string `valid:"required~Required"` -} - -type GetByModuleNameRequest struct { - ModuleName string `valid:"required~Required"` -} - -type BroadcastEventRequest struct { - ModuleNames []string `valid:"required~Required"` - Event string `valid:"required~Required"` - Payload json2.RawMessage -} - -type CreateUpdateConfigRequest struct { - entity.Config - Unsafe bool -} diff --git a/domain/response.go b/domain/response.go deleted file mode 100644 index e41c433..0000000 --- a/domain/response.go +++ /dev/null @@ -1,59 +0,0 @@ -package domain - -import ( - "time" - - "github.com/integration-system/isp-lib/v2/config/schema" - "github.com/integration-system/isp-lib/v2/structure" - "isp-config-service/entity" -) - -type DeleteResponse struct { - Deleted int -} - -type ModuleInfo struct { - Id string - Name string - Active bool - CreatedAt time.Time - LastConnectedAt time.Time - LastDisconnectedAt time.Time - Configs []ConfigModuleInfo - ConfigSchema *schema.Schema - Status []Connection - RequiredModules []ModuleDependency -} - -type Connection struct { - LibVersion string - Version string - Address structure.AddressConfiguration - Endpoints []structure.EndpointDescriptor - EstablishedAt time.Time -} - -type ModuleDependency struct { - Id string - Name string - Required bool -} - -type CommonConfigLinks map[string][]string - -type CompiledConfigResponse map[string]interface{} - -type DeleteCommonConfigResponse struct { - Deleted bool - Links CommonConfigLinks -} - -type CreateUpdateConfigResponse struct { - ErrorDetails map[string]string - Config *ConfigModuleInfo -} - -type ConfigModuleInfo struct { - entity.Config - Valid bool -} diff --git a/entity/config.go b/entity/config.go deleted file mode 100644 index c78fe0a..0000000 --- a/entity/config.go +++ /dev/null @@ -1,46 +0,0 @@ -package entity - -import ( - "time" - - "github.com/integration-system/isp-lib/v2/config/schema" -) - -type ConfigData map[string]interface{} - -type Config struct { - //nolint - tableName string `pg:"?db_schema.configs" json:"-"` - Id string `json:"id"` - Name string `json:"name" valid:"required~Required"` - CommonConfigs []string `json:"commonConfigs" pg:",array"` - Description string `json:"description"` - ModuleId string `json:"moduleId" valid:"required~Required"` - Version int32 `json:"version" pg:",null"` - Active bool `json:"active" pg:",null"` - CreatedAt time.Time `json:"createdAt" pg:",null"` - UpdatedAt time.Time `json:"updatedAt" pg:",null"` - Data ConfigData `json:"data,omitempty" pg:",notnull"` -} - -type CommonConfig struct { - //nolint - tableName string `pg:"?db_schema.common_configs" json:"-"` - Id string `json:"id"` - Name string `json:"name" valid:"required~Required"` - Description string `json:"description"` - CreatedAt time.Time `json:"createdAt" pg:",null"` - UpdatedAt time.Time `json:"updatedAt" pg:",null"` - Data ConfigData `json:"data" pg:",notnull"` -} - -type ConfigSchema struct { - //nolint - tableName string `pg:"?db_schema.config_schemas" json:"-"` - Id string `json:"id"` - Version string - ModuleId string - Schema schema.Schema - CreatedAt time.Time - UpdatedAt time.Time -} diff --git a/entity/module.go b/entity/module.go index 196c470..49ac4c7 100644 --- a/entity/module.go +++ b/entity/module.go @@ -1,13 +1,13 @@ package entity -import "time" +import ( + "time" +) type Module struct { - //nolint - tableName string `pg:"?db_schema.modules" json:"-"` - Id string `json:"id"` - Name string `json:"name" valid:"required~Required"` - CreatedAt time.Time `json:"createdAt" pg:",null"` - LastConnectedAt time.Time `json:"lastConnectedAt"` - LastDisconnectedAt time.Time `json:"lastDisconnectedAt"` + Id string + Name string + LastConnectedAt time.Time + LastDisconnectedAt time.Time + Created time.Time } diff --git a/entity/version_config.go b/entity/version_config.go deleted file mode 100644 index 8810334..0000000 --- a/entity/version_config.go +++ /dev/null @@ -1,15 +0,0 @@ -package entity - -import ( - "time" -) - -type VersionConfig struct { - //nolint - tableName string `pg:"?db_schema.version_config" json:"-"` - Id string - ConfigId string - ConfigVersion int32 - Data ConfigData `json:"data,omitempty" pg:",notnull"` - CreatedAt time.Time -} diff --git a/go.mod b/go.mod index b929085..a9e5c42 100644 --- a/go.mod +++ b/go.mod @@ -1,103 +1,96 @@ module isp-config-service -go 1.19 +go 1.22 require ( - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d - github.com/cenkalti/backoff/v4 v4.2.0 - github.com/go-pg/pg/v9 v9.2.1 - github.com/hashicorp/raft v1.3.11 - github.com/hashicorp/raft-boltdb/v2 v2.2.2 - github.com/integration-system/bellows v1.0.1 - github.com/integration-system/isp-etp-go/v2 v2.1.3 - github.com/integration-system/isp-lib/v2 v2.11.0 - github.com/integration-system/isp-log v1.2.0 - github.com/integration-system/net-mux v1.0.1 - github.com/integration-system/swag v1.7.115 - github.com/json-iterator/go v1.1.12 - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 + github.com/jmoiron/sqlx v1.4.0 github.com/pkg/errors v0.9.1 - github.com/satori/go.uuid v1.2.0 - github.com/xeipuuv/gojsonschema v1.2.0 - google.golang.org/grpc v1.51.0 + github.com/pressly/goose/v3 v3.20.0 + github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 + github.com/rqlite/rqlite/v8 v8.23.4 + github.com/tidwall/gjson v1.17.1 + github.com/txix-open/etp/v3 v3.0.0 + github.com/txix-open/isp-kit v1.31.2 ) require ( - github.com/KyleBanks/depth v1.2.1 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/armon/go-metrics v0.4.1 // indirect - github.com/boltdb/bolt v1.3.1 // indirect - github.com/buaazp/fasthttprouter v0.1.1 // indirect - github.com/codemodus/kace v0.5.1 // indirect - github.com/fatih/color v1.13.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/spec v0.20.7 // indirect - github.com/go-openapi/swag v0.22.3 // indirect - github.com/go-pg/zerochecker v0.2.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-hclog v1.3.1 // indirect + github.com/armon/go-metrics v0.5.3 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-msgpack v0.5.5 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-uuid v1.0.1 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/integration-system/go-cmp v0.0.0-20190131081942-ac5582987a2f // indirect - github.com/integration-system/jsonschema v1.0.1 // indirect - github.com/jinzhu/inflection v1.0.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.15.12 // indirect - github.com/lib/pq v1.10.7 // indirect - github.com/magiconair/properties v1.8.6 // indirect + github.com/hashicorp/go-msgpack v1.1.5 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/raft v1.6.1 // indirect + github.com/iancoleman/strcase v0.3.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/ginkgo v1.15.0 // indirect - github.com/onsi/gomega v1.10.5 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.6 // indirect - github.com/pressly/goose v2.7.0+incompatible // indirect - github.com/profefe/profefe v0.0.0-20210412073930-18342f204d75 // indirect - github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/segmentio/asm v1.2.0 // indirect - github.com/segmentio/encoding v0.3.6 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect - github.com/spf13/afero v1.9.3 // indirect - github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.14.0 // indirect - github.com/subosito/gotenv v1.4.1 // indirect - github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a // indirect - github.com/swaggo/http-swagger v1.3.3 // indirect - github.com/swaggo/swag v1.8.8 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.43.0 // indirect - github.com/vmihailenco/bufpool v0.1.11 // indirect - github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect - github.com/vmihailenco/tagparser v0.1.2 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/crypto v0.3.0 // indirect - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/net v0.2.0 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/text v0.4.0 // indirect - golang.org/x/tools v0.3.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.53.0 // indirect + github.com/prometheus/procfs v0.14.0 // indirect + github.com/rqlite/go-sqlite3 v1.32.0 // indirect + github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 // indirect + github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd // indirect + github.com/sethvargo/go-retry v0.2.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/txix-open/bellows v1.2.0 // indirect + github.com/txix-open/jsonschema v1.2.0 // indirect + github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + go.etcd.io/bbolt v1.3.9 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.51.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/sdk v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - mellium.im/sasl v0.3.0 // indirect - nhooyr.io/websocket v1.8.7 // indirect + nhooyr.io/websocket v1.8.11 // indirect +) + +replace ( + github.com/armon/go-metrics => github.com/hashicorp/go-metrics v0.5.1 + golang.org/x/text => golang.org/x/text v0.3.8 ) diff --git a/go.sum b/go.sum deleted file mode 100644 index cb655dd..0000000 --- a/go.sum +++ /dev/null @@ -1,905 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/clickhouse-go v1.4.0/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= -github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= -github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= -github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.29.9/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/buaazp/fasthttprouter v0.1.1 h1:4oAnN0C3xZjylvZJdP35cxfclyn4TYkW6Y+DSvS+h8Q= -github.com/buaazp/fasthttprouter v0.1.1/go.mod h1:h/Ap5oRVLeItGKTVBb+heQPks+HdIUtGmI4H5WCYijM= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/codemodus/kace v0.5.1 h1:4OCsBlE2c/rSJo375ggfnucv9eRzge/U5LrrOZd47HA= -github.com/codemodus/kace v0.5.1/go.mod h1:coddaHoX1ku1YFSe4Ip0mL9kQjJvKkzb9CfIdG1YR04= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= -github.com/go-openapi/spec v0.20.7 h1:1Rlu/ZrOCCob0n+JKKJAWhNWMPW8bOZRg8FJaY+0SKI= -github.com/go-openapi/spec v0.20.7/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-pg/pg/v9 v9.2.1 h1:4rWNJkj+aPuDFqgieTzNhHBuYaXREh3yaB9NlBerFys= -github.com/go-pg/pg/v9 v9.2.1/go.mod h1:fG8qbL+ei4e/fCZLHK+Z+/7b9B+pliZtbpaucG4/YNQ= -github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= -github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.3.1 h1:vDwF1DFNZhntP4DAjuTpOw3uEgMUpXh1pB5fW9DqHpo= -github.com/hashicorp/go-hclog v1.3.1/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= -github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A= -github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= -github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= -github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= -github.com/hashicorp/raft-boltdb/v2 v2.2.2 h1:rlkPtOllgIcKLxVT4nutqlTH2NRFn+tO1wwZk/4Dxqw= -github.com/hashicorp/raft-boltdb/v2 v2.2.2/go.mod h1:N8YgaZgNJLpZC+h+by7vDu5rzsRgONThTEeUS3zWbfY= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/integration-system/bellows v1.0.1 h1:7g1lhD/6EcvsSo9iuiFqw98XGS78Sko6kUKPzfte/SY= -github.com/integration-system/bellows v1.0.1/go.mod h1:qHPg1ptN3AHHr0f/wvoicfA2LOCprOyrbXnIa5QIRe0= -github.com/integration-system/go-cmp v0.0.0-20190131081942-ac5582987a2f h1:52UOJguw5yqe1y2JjmyVxGcZf3kv2BQeUodR363MkiQ= -github.com/integration-system/go-cmp v0.0.0-20190131081942-ac5582987a2f/go.mod h1:w+dmaWojuIwNujSYg6qt1wKHhSdorLOivCEl/XuLzbM= -github.com/integration-system/isp-etp-go/v2 v2.1.3 h1:vcHP/PE7p1G3rxqoVKrKtXzoSxFA4ryxigd+zPkAb3w= -github.com/integration-system/isp-etp-go/v2 v2.1.3/go.mod h1:ZpB+FIH3hy7md+f8ylVSQx5zwZVjczhkc45f1B2z9nw= -github.com/integration-system/isp-lib/v2 v2.11.0 h1:5vlgSgTvPY2Q3/ucTZ/572GPeJsVGqz+y74VhKZ2LKU= -github.com/integration-system/isp-lib/v2 v2.11.0/go.mod h1:9u47SFT0sTAZM5rkqeV88HNTiExgZWrRFoAQpjC68Wc= -github.com/integration-system/isp-log v1.2.0 h1:kYAP8LH2ttDk9lElKNoxGvLAKh94kesfnAkQVCjbc5s= -github.com/integration-system/isp-log v1.2.0/go.mod h1:ri19IsCkLbxPWS1KFNHlLpg88IdYF5EHm8JH+xULLQ0= -github.com/integration-system/jsonschema v1.0.1 h1:nR78BEDG9djX2uXQ1YP66a/9yQOMmp2HFmw8VPUdAa8= -github.com/integration-system/jsonschema v1.0.1/go.mod h1:EoqDH7RmC8Epg5EgjboIxDo7Udv3WX2exGqtIVB4hI0= -github.com/integration-system/net-mux v1.0.1 h1:UKHdt1UxNJXwroeLdhYQeUah/aizcqx6hS1SOdfx7pg= -github.com/integration-system/net-mux v1.0.1/go.mod h1:0iEskyZB58/VwtFu8BATYMmPse2ZU9WWv6agidMEAs0= -github.com/integration-system/swag v1.7.115 h1:MnKJO3vifqJuc2SL8BEagJzzTOKoFZOVdrLL4JXdnb8= -github.com/integration-system/swag v1.7.115/go.mod h1:ELPtyiM3C6kzIFEmhEkls73vQ7ZKyH4fbQEkrOuBYcc= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= -github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= -github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= -github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= -github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose v2.7.0+incompatible h1:PWejVEv07LCerQEzMMeAtjuyCKbyprZ/LBa6K5P0OCQ= -github.com/pressly/goose v2.7.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= -github.com/profefe/profefe v0.0.0-20210412073930-18342f204d75 h1:L2ySMPFVEjnQA+CcPdG4/j+R+7tH95MtB3Y0Wynktes= -github.com/profefe/profefe v0.0.0-20210412073930-18342f204d75/go.mod h1:64I4nyfqy58IxcBlDQ+SgtwdAF0m9yVxfCiVKVwag/k= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.9/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rs/xid v1.2.2-0.20200205151950-d13a6085d55c/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= -github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= -github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/segmentio/encoding v0.1.15/go.mod h1:RWhr02uzMB9gQC1x+MfYxedtmBibb9cZ6Vv9VxRSSbw= -github.com/segmentio/encoding v0.3.6 h1:E6lVLyDPseWEulBmCmAKPanDd3jiyGDo5gMcugCRwZQ= -github.com/segmentio/encoding v0.3.6/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= -github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= -github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a h1:kAe4YSu0O0UFn1DowNo2MY5p6xzqtJ/wQ7LZynSvGaY= -github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= -github.com/swaggo/http-swagger v1.3.3 h1:Hu5Z0L9ssyBLofaama21iYaF2VbWyA8jdohaaCGpHsc= -github.com/swaggo/http-swagger v1.3.3/go.mod h1:sE+4PjD89IxMPm77FnkDz0sdO+p5lbXzrVWT6OTVVGo= -github.com/swaggo/swag v1.8.8 h1:/GgJmrJ8/c0z4R4hoEPZ5UeEhVGdvsII4JbVDLbR7Xc= -github.com/swaggo/swag v1.8.8/go.mod h1:ezQVUUhly8dludpVk+/PuwJWvLLanB13ygV5Pr9enSk= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.43.0 h1:Gy4sb32C98fbzVWZlTM1oTMdLWGyvxR03VhM6cBIU4g= -github.com/valyala/fasthttp v1.43.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= -github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= -github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= -github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= -github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= -github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= -golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= -google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= -mellium.im/sasl v0.3.0 h1:0qoaTCTo5Py7u/g0cBIQZcMOgG/5LM71nshbXwznBh8= -mellium.im/sasl v0.3.0/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= -nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= -nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/helper/handler.go b/helper/handler.go deleted file mode 100644 index 049eb51..0000000 --- a/helper/handler.go +++ /dev/null @@ -1,68 +0,0 @@ -//nolint:lll -package helper - -import ( - "github.com/integration-system/isp-lib/v2/structure" - "isp-config-service/controller" - "isp-config-service/domain" - "isp-config-service/entity" -) - -type Handlers struct { - // ===== MODULE ===== - DeleteModules func(identities []string) (*domain.DeleteResponse, error) `method:"delete_module" group:"module" inner:"true"` - GetModulesInfo func() ([]domain.ModuleInfo, error) `method:"get_modules_info" group:"module" inner:"true"` - GetModuleByName func(req domain.GetByModuleNameRequest) (*entity.Module, error) `method:"get_by_name" group:"module" inner:"true"` - BroadcastEvent func(req domain.BroadcastEventRequest) error `method:"broadcast_event" group:"module" inner:"true"` - - // ===== CONFIG ===== - GetActiveConfigByModuleName func(request domain.GetByModuleNameRequest) (*entity.Config, error) `method:"get_active_config_by_module_name" group:"config" inner:"true"` - GetConfigsByModuleId func(request domain.GetByModuleIdRequest) ([]domain.ConfigModuleInfo, error) `method:"get_configs_by_module_id" group:"config" inner:"true"` - CreateUpdateConfig func(config domain.CreateUpdateConfigRequest) (*domain.ConfigModuleInfo, error) `method:"create_update_config" group:"config" inner:"true"` - MarkConfigAsActive func(identity domain.ConfigIdRequest) (*entity.Config, error) `method:"mark_config_as_active" group:"config" inner:"true"` - DeleteConfig func(identities []string) (*domain.DeleteResponse, error) `method:"delete_config" group:"config" inner:"true"` - DeleteVersion func(identity domain.ConfigIdRequest) (*domain.DeleteResponse, error) `method:"delete_version" group:"config" inner:"true"` - GetAllVersion func(identity domain.ConfigIdRequest) ([]entity.VersionConfig, error) `method:"get_all_version" group:"config" inner:"true"` - GetConfigByID func(identity domain.ConfigIdRequest) (entity.Config, error) `method:"get_config_by_id" group:"config" inner:"true"` - - // ===== COMMON CONFIG ===== - GetCommonConfigs func(identities []string) []entity.CommonConfig `method:"get_configs" group:"common_config" inner:"true"` - CreateUpdateCommonConfig func(config entity.CommonConfig) (*entity.CommonConfig, error) `method:"create_update_config" group:"common_config" inner:"true"` - DeleteCommonConfig func(req domain.ConfigIdRequest) (*domain.DeleteCommonConfigResponse, error) `method:"delete_config" group:"common_config" inner:"true"` - CompileConfigs func(req domain.CompileConfigsRequest) domain.CompiledConfigResponse `method:"compile" group:"common_config" inner:"true"` - GetLinks func(req domain.ConfigIdRequest) domain.CommonConfigLinks `method:"get_links" group:"common_config" inner:"true"` - - // ===== ROUTING ===== - GetRoutes func() ([]structure.BackendDeclaration, error) `method:"get_routes" group:"routing" inner:"true"` - - // ===== SCHEMA ===== - GetSchemaByModuleId func(domain.GetByModuleIdRequest) (*entity.ConfigSchema, error) `method:"get_by_module_id" group:"schema" inner:"true"` -} - -func GetHandlers() *Handlers { - return &Handlers{ - DeleteModules: controller.Module.DeleteModules, - GetModulesInfo: controller.Module.GetModulesAggregatedInfo, - GetModuleByName: controller.Module.GetModuleByName, - BroadcastEvent: controller.Module.BroadcastEvent, - - GetActiveConfigByModuleName: controller.Config.GetActiveConfigByModuleName, - GetConfigsByModuleId: controller.Config.GetConfigsByModuleId, - CreateUpdateConfig: controller.Config.CreateUpdateConfig, - MarkConfigAsActive: controller.Config.MarkConfigAsActive, - DeleteConfig: controller.Config.DeleteConfigs, - DeleteVersion: controller.Config.DeleteConfigVersion, - GetAllVersion: controller.Config.GetAllVersion, - GetConfigByID: controller.Config.GetConfigById, - - GetCommonConfigs: controller.CommonConfig.GetConfigs, - CreateUpdateCommonConfig: controller.CommonConfig.CreateUpdateConfig, - DeleteCommonConfig: controller.CommonConfig.DeleteConfigs, - CompileConfigs: controller.CommonConfig.CompileConfigs, - GetLinks: controller.CommonConfig.GetLinks, - - GetRoutes: controller.Routes.GetRoutes, - - GetSchemaByModuleId: controller.Schema.GetByModuleId, - } -} diff --git a/holder/holder.go b/holder/holder.go deleted file mode 100644 index d7bd0f3..0000000 --- a/holder/holder.go +++ /dev/null @@ -1,14 +0,0 @@ -package holder - -import ( - "net/http" - - etp "github.com/integration-system/isp-etp-go/v2" - "isp-config-service/cluster" -) - -var ( - ClusterClient *cluster.Client - EtpServer etp.Server - HTTPServer *http.Server -) diff --git a/main.go b/main.go index e335b8a..caa7d2f 100644 --- a/main.go +++ b/main.go @@ -1,262 +1,46 @@ package main import ( - "context" - "fmt" - "net" - "net/http" - "os" - "os/signal" - "syscall" - "time" - - "github.com/integration-system/isp-etp-go/v2" - "github.com/integration-system/isp-lib/v2/backend" - "github.com/integration-system/isp-lib/v2/bootstrap" - "github.com/integration-system/isp-lib/v2/config" - "github.com/integration-system/isp-lib/v2/metric" - "github.com/integration-system/isp-lib/v2/structure" - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - mux "github.com/integration-system/net-mux" - "isp-config-service/cluster" - "isp-config-service/codes" + "github.com/txix-open/isp-kit/bootstrap" + "github.com/txix-open/isp-kit/shutdown" "isp-config-service/conf" - "isp-config-service/controller" - "isp-config-service/helper" - "isp-config-service/holder" - "isp-config-service/model" - "isp-config-service/raft" - "isp-config-service/service" - "isp-config-service/store" - "isp-config-service/store/state" - "isp-config-service/subs" -) - -const ( - moduleName = "config" - defaultWsConnectionReadLimit int64 = 4 << 20 // 4 MB + "isp-config-service/routes" + "isp-config-service/service/startup" ) var ( - version = "0.1.0" - - muxer mux.Mux + version = "1.0.0" ) -func init() { - config.InitConfig(&conf.Configuration{}) - if utils.LOG_LEVEL != "" { - _ = log.SetLevel(utils.LOG_LEVEL) - } -} - -// @title ISP configuration service -// @version 2.4.1 -// @description Сервис управления конфигурацией модулей ISP кластера +// @title isp-config-service +// @version 1.0.0 +// @description Модуль управления конфигурациями // @license.name GNU GPL v3.0 -// @host localhost:9003 -// @BasePath /api/config +// @host localhost:9000 +// @BasePath /api/isp-config-service //go:generate swag init --parseDependency -//go:generate rm -f docs/swagger.json -func main() { - cfg := config.Get().(*conf.Configuration) - handlers := helper.GetHandlers() - endpoints := backend.GetEndpoints(cfg.ModuleName, handlers) - address := cfg.GrpcOuterAddress - if address.IP == "" { - ip, err := getOutboundIP() - if err != nil { - panic(err) - } - address.IP = ip - } - declaration := structure.BackendDeclaration{ - ModuleName: cfg.ModuleName, - Version: version, - LibVersion: bootstrap.LibraryVersion, - Endpoints: endpoints, - Address: address, - } - - connectionReadLimit := defaultWsConnectionReadLimit - if cfg.WS.WsConnectionReadLimitKB > 0 { - //nolint:gomnd - connectionReadLimit = cfg.WS.WsConnectionReadLimitKB << 10 - } - cluster.WsConnectionReadLimit = connectionReadLimit - - cfg.Database.Password = fmt.Sprintf("%s%s", cfg.Database.Password, os.Getenv("DB_PASSPART")) - model.DbClient.ReceiveConfiguration(cfg.Database) - - httpListener, raftListener, err := initMultiplexer(cfg.WS.Rest) - if err != nil { - log.Fatalf(codes.InitMuxError, "init mux: %v", err) - } - - _, raftStore := initRaft(raftListener, cfg.Cluster, declaration) - initWebsocket(connectionReadLimit, httpListener, raftStore) - initGrpc(cfg.WS.Grpc, raftStore) - - metric.InitProfiling(cfg.ModuleName) - metric.InitCollectors(cfg.Metrics, structure.MetricConfiguration{}) - metric.InitHttpServer(cfg.Metrics) - - gracefulShutdown() -} - -func initMultiplexer(addressConfiguration structure.AddressConfiguration) (net.Listener, net.Listener, error) { - outerAddr, err := net.ResolveTCPAddr("tcp4", addressConfiguration.GetAddress()) - if err != nil { - return nil, nil, fmt.Errorf("resolve outer address: %v", err) - } - tcpListener, err := net.ListenTCP("tcp4", outerAddr) - if err != nil { - return nil, nil, fmt.Errorf("create tcp transport: %v", err) - } - - muxer = mux.New(tcpListener) - httpListener := muxer.Match(mux.HTTP1()) - raftListener := muxer.Match(mux.Any()) - - go func() { - if err := muxer.Serve(); err != nil { - log.Fatalf(codes.InitMuxError, "serve mux: %v", err) - } - }() - return httpListener, raftListener, nil -} +//go:generate rm -f docs/swagger.json docs/docs.go -func initWebsocket(connectionReadLimit int64, listener net.Listener, raftStore *store.Store) { - etpConfig := etp.ServerConfig{ - InsecureSkipVerify: true, - ConnectionReadLimit: connectionReadLimit, - } - etpServer := etp.NewServer(context.Background(), etpConfig) - subs.NewSocketEventHandler(etpServer, raftStore).SubscribeAll() - - mux := http.NewServeMux() - mux.HandleFunc("/isp-etp/", etpServer.ServeHttp) - httpServer := &http.Server{Handler: mux} - go func() { - if err := httpServer.Serve(listener); err != nil && err != http.ErrServerClosed { - log.Fatalf(codes.StartHttpServerError, "http server closed: %v", err) - } - }() - holder.EtpServer = etpServer - holder.HTTPServer = httpServer -} - -func initRaft(listener net.Listener, clusterCfg conf.ClusterConfiguration, - declaration structure.BackendDeclaration) (*cluster.Client, *store.Store) { - raftState, err := store.NewStateFromRepository() - if err != nil { - log.Fatal(codes.RestoreFromRepositoryError, err) - return nil, nil - } - raftStore := store.NewStateStore(raftState) - r, err := raft.NewRaft(listener, clusterCfg, raftStore) - if err != nil { - log.Fatalf(codes.InitRaftError, "unable to create raft server. %v", err) - return nil, nil - } - clusterClient := cluster.NewRaftClusterClient(r, declaration, func(address string) { - go raftStore.VisitState(func(s state.WritableState) { - cfg := config.Get().(*conf.Configuration) - addr, err := net.ResolveTCPAddr("tcp", address) - if err != nil { - panic(err) // must never occurred - } - port := cfg.GrpcOuterAddress.Port - addressConfiguration := structure.AddressConfiguration{Port: port, IP: addr.IP.String()} - back := structure.BackendDeclaration{ModuleName: cfg.ModuleName, Address: addressConfiguration} - log.WithMetadata(map[string]interface{}{"declaration": back}). - Infof(codes.LeaderManualDeleteLeader, "manually delete disconnected leader's declaration") - service.ClusterMesh.HandleDeleteBackendDeclarationCommand(back, s) - }) +func main() { + boot := bootstrap.New(version, conf.Remote{}, routes.EndpointDescriptors()) + app := boot.App + logger := app.Logger() + + startup := startup.New(boot) + app.AddRunners(startup) + app.AddClosers(startup) + + shutdown.On(func() { + logger.Info(app.Context(), "starting shutdown") + app.Shutdown() + logger.Info(app.Context(), "shutdown completed") }) - holder.ClusterClient = clusterClient - - if clusterCfg.BootstrapCluster { - err = r.BootstrapCluster() // err can be ignored - if err != nil { - log.Errorf(codes.BootstrapClusterError, "bootstrap cluster. %v", err) - } - } - return clusterClient, raftStore -} - -func initGrpc(bindAddress structure.AddressConfiguration, raftStore *store.Store) { - controller.Routes = controller.NewRoutes(raftStore) - controller.Schema = controller.NewSchema(raftStore) - controller.Module = controller.NewModule(raftStore) - controller.Config = controller.NewConfig(raftStore) - controller.CommonConfig = controller.NewCommonConfig(raftStore) - defaultService := backend.GetDefaultService(moduleName, helper.GetHandlers()) - backend.StartBackendGrpcServer(bindAddress, defaultService) -} - -func gracefulShutdown() { - const ( - gracefulTimeout = 10 * time.Second - terminateTimeout = 11 * time.Second - ) - - signalsCh := make(chan os.Signal, 1) - signal.Notify(signalsCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) - sig := <-signalsCh - log.Infof(0, "received signal '%s'", sig) - - finishedCh := make(chan struct{}) - go func() { - select { - case <-time.After(terminateTimeout): - log.Fatal(0, "exit timeout reached: terminating") - case <-finishedCh: - // ok - case sig := <-signalsCh: - log.Fatalf(0, "received duplicate exit signal '%s': terminating", sig) - } - }() - - ctx, cancel := context.WithTimeout(context.Background(), gracefulTimeout) - defer cancel() - onShutdown(ctx) - finishedCh <- struct{}{} - log.Info(0, "application exited normally") -} - -func onShutdown(ctx context.Context) { - backend.StopGrpcServer() - - if err := holder.HTTPServer.Shutdown(ctx); err != nil { - log.Warnf(codes.ShutdownHttpServerError, "http server shutdown err: %v", err) - } else { - log.Info(codes.ShutdownHttpServerInfo, "http server shutdown success") - } - holder.EtpServer.Close() - - if err := holder.ClusterClient.Shutdown(); err != nil { - log.Warnf(codes.RaftShutdownError, "raft shutdown err: %v", err) - } else { - log.Info(codes.RaftShutdownInfo, "raft shutdown success") - } - - if err := model.DbClient.Close(); err != nil { - log.Warnf(0, "database shutdown err: %v", err) - } - - _ = muxer.Close() -} -func getOutboundIP() (string, error) { - conn, err := net.Dial("udp", "9.9.9.9:80") + err := app.Run() if err != nil { - return "", err + boot.Fatal(err) } - defer conn.Close() - return conn.LocalAddr().(*net.UDPAddr).IP.To4().String(), nil } diff --git a/migrations/00001_2018-04-05_create_schema.sql b/migrations/00001_2018-04-05_create_schema.sql deleted file mode 100644 index 091ab13..0000000 --- a/migrations/00001_2018-04-05_create_schema.sql +++ /dev/null @@ -1,5 +0,0 @@ --- +goose Up -CREATE schema if NOT EXISTS "config_service"; - --- +goose Down -DROP schema if EXISTS config_service CASCADE; \ No newline at end of file diff --git a/migrations/00002_2018-04-05_create_instances_table.sql b/migrations/00002_2018-04-05_create_instances_table.sql deleted file mode 100644 index c2ef082..0000000 --- a/migrations/00002_2018-04-05_create_instances_table.sql +++ /dev/null @@ -1,47 +0,0 @@ --- +goose Up -CREATE TABLE instances ( - id serial4 NOT NULL PRIMARY KEY, - uuid UUID NOT NULL, - "name" varchar(255) NOT NULL, - created_at timestamp DEFAULT (now() at time zone 'utc') NOT NULL -); - --- +goose StatementBegin -CREATE OR REPLACE FUNCTION update_created_modified_column_date() - RETURNS TRIGGER AS -$body$ -BEGIN - IF TG_OP = 'UPDATE' THEN - NEW.created_at = OLD.created_at; - NEW.updated_at = (now() at time zone 'utc'); - ELSIF TG_OP = 'INSERT' THEN - NEW.updated_at = (now() at time zone 'utc'); - END IF; - RETURN NEW; -END; -$body$ LANGUAGE plpgsql; --- +goose StatementEnd - --- +goose StatementBegin -CREATE OR REPLACE FUNCTION update_created_column_date() - RETURNS TRIGGER AS -$body$ -BEGIN - IF TG_OP = 'UPDATE' THEN - NEW.created_at = OLD.created_at; - ELSIF TG_OP = 'INSERT' THEN - NEW.created_at = (now() at time zone 'utc'); - END IF; - RETURN NEW; -END; -$body$ LANGUAGE plpgsql; --- +goose StatementEnd - -CREATE TRIGGER update_customer_modtime BEFORE INSERT OR UPDATE ON instances - FOR EACH ROW EXECUTE PROCEDURE update_created_column_date(); - -ALTER TABLE instances ADD CONSTRAINT "UQ_instances_uuid" UNIQUE ("uuid"); - - --- +goose Down -DROP TABLE instances; diff --git a/migrations/00003_2018-04-05_create_modules_table.sql b/migrations/00003_2018-04-05_create_modules_table.sql deleted file mode 100644 index 43f4ed5..0000000 --- a/migrations/00003_2018-04-05_create_modules_table.sql +++ /dev/null @@ -1,24 +0,0 @@ --- +goose Up -CREATE TABLE modules ( - id serial4 NOT NULL PRIMARY KEY, - instance_id int4 NOT NULL, - "name" varchar(255) NOT NULL, - "active" bool DEFAULT true, - created_at timestamp DEFAULT (now() at time zone 'utc') NOT NULL, - last_connected_at timestamp, - last_disconnected_at timestamp -); - -ALTER TABLE modules - ADD CONSTRAINT "FK_modules_instanceId_instances_id" - FOREIGN KEY ("instance_id") REFERENCES instances ("id") - ON DELETE CASCADE ON UPDATE CASCADE; -CREATE INDEX IX_modules_instanceId ON modules USING hash (instance_id); - -CREATE TRIGGER update_customer_modtime BEFORE INSERT OR UPDATE ON modules - FOR EACH ROW EXECUTE PROCEDURE update_created_column_date(); - -ALTER TABLE modules ADD CONSTRAINT "UQ_modules_instanceId_name" UNIQUE ("instance_id", "name"); - --- +goose Down -DROP TABLE modules; diff --git a/migrations/00004_2018-04-05_create_configs_table.sql b/migrations/00004_2018-04-05_create_configs_table.sql deleted file mode 100644 index 672cf30..0000000 --- a/migrations/00004_2018-04-05_create_configs_table.sql +++ /dev/null @@ -1,70 +0,0 @@ --- +goose Up -CREATE TABLE configs ( - id serial8 NOT NULL PRIMARY KEY, - module_id int4 NOT NULL, - "version" int4 NOT NULL, - "active" bool DEFAULT false, - created_at timestamp DEFAULT (now() at time zone 'utc') NOT NULL, - updated_at timestamp DEFAULT (now() at time zone 'utc') NOT NULL, - "data" jsonb NOT NULL DEFAULT '{}' -); - -ALTER TABLE configs - ADD CONSTRAINT "FK_configs_moduleId_modules_id" - FOREIGN KEY ("module_id") REFERENCES modules ("id") - ON DELETE CASCADE ON UPDATE CASCADE; - -CREATE INDEX IX_configs_moduleId ON configs USING hash (module_id); - --- +goose StatementBegin -CREATE OR REPLACE FUNCTION deactivate_config() - RETURNS TRIGGER AS -$body$ -DECLARE - last_version integer; -BEGIN - NEW.updated_at = (now() at time zone 'utc'); - IF TG_OP = 'INSERT' - THEN - EXECUTE format(' - SELECT version - FROM %I.configs - WHERE module_id = ' || NEW.module_id || ' - ORDER BY created_at DESC - LIMIT 1; - ', TG_TABLE_SCHEMA) INTO last_version; - IF last_version IS NOT NULL - THEN - NEW.version = last_version + 1; - ELSE - NEW.version = 1; - END IF; - ELSE - NEW.version = OLD.version; - NEW.created_at = OLD.created_at; - END IF; - - IF NEW.active = TRUE - THEN - EXECUTE format(' - UPDATE %I.configs - SET active = FALSE, - updated_at = (now() at time zone ''utc'') - WHERE active = TRUE AND ' || NEW.id || ' != id AND module_id = ' || NEW.module_id || '; - ', TG_TABLE_SCHEMA); - END IF; - RETURN NEW; -END; -$body$ LANGUAGE plpgsql; --- +goose StatementEnd - -CREATE TRIGGER "deactivate_config" - BEFORE INSERT OR UPDATE ON configs - FOR EACH ROW EXECUTE PROCEDURE deactivate_config(); - -CREATE TRIGGER update_config_create_time BEFORE INSERT OR UPDATE ON configs - FOR EACH ROW EXECUTE PROCEDURE update_created_column_date(); - --- +goose Down -DROP TABLE configs; -DROP FUNCTION deactivate_config; diff --git a/migrations/00005_2018-07-31_add_name_for_configs.sql b/migrations/00005_2018-07-31_add_name_for_configs.sql deleted file mode 100644 index 4f29754..0000000 --- a/migrations/00005_2018-07-31_add_name_for_configs.sql +++ /dev/null @@ -1,9 +0,0 @@ --- +goose Up -ALTER TABLE configs - ADD COLUMN name VARCHAR(255) NOT NULL DEFAULT 'unnamed', - ADD COLUMN description TEXT; - --- +goose Down -ALTER TABLE configs - DROP COLUMN name, - DROP COLUMN description; diff --git a/migrations/00006_2018-07-31_config_sheme_table.sql b/migrations/00006_2018-07-31_config_sheme_table.sql deleted file mode 100644 index f2fccb8..0000000 --- a/migrations/00006_2018-07-31_config_sheme_table.sql +++ /dev/null @@ -1,23 +0,0 @@ --- +goose Up -CREATE TABLE config_schemas ( - id SERIAL4 PRIMARY KEY, - module_id INTEGER NOT NULL, - version VARCHAR(255) NOT NULL, - schema JSONB NOT NULL DEFAULT '{}', - created_at TIMESTAMP NOT NULL DEFAULT (now() at time zone 'utc'), - updated_at TIMESTAMP NOT NULL DEFAULT (now() at time zone 'utc'), - CONSTRAINT FK_module_id__module_id FOREIGN KEY (module_id) - REFERENCES modules (id) ON UPDATE CASCADE ON DELETE CASCADE -); - -CREATE TRIGGER update_schema_create_time - BEFORE INSERT OR UPDATE - ON config_schemas - FOR EACH ROW EXECUTE PROCEDURE update_created_modified_column_date(); - -CREATE INDEX IX_schemes_moduleId - ON config_schemas - USING hash (module_id); - --- +goose Down -DROP TABLE config_schemas; diff --git a/migrations/00007_2019-10-08_init_v2.sql b/migrations/00007_2019-10-08_init_v2.sql deleted file mode 100644 index 32dc4ef..0000000 --- a/migrations/00007_2019-10-08_init_v2.sql +++ /dev/null @@ -1,46 +0,0 @@ --- +goose Up - -DROP TABLE instances CASCADE; - -DROP TRIGGER update_config_create_time ON configs; -DROP TRIGGER deactivate_config on configs; -DROP TRIGGER update_schema_create_time ON config_schemas; -DROP TRIGGER update_customer_modtime ON modules; - - -ALTER TABLE configs - DROP CONSTRAINT "FK_configs_moduleId_modules_id"; - -ALTER TABLE config_schemas - DROP CONSTRAINT "fk_module_id__module_id"; - -ALTER TABLE configs - ALTER COLUMN id TYPE varchar(255), - ALTER COLUMN module_id TYPE varchar(255), - ADD COLUMN common_configs varchar(255)[] NOT NULL DEFAULT '{}'; - -ALTER TABLE config_schemas - ALTER COLUMN module_id TYPE varchar(255), - ALTER COLUMN id TYPE varchar(255); - -ALTER TABLE modules - ALTER COLUMN id TYPE varchar(255), - DROP COLUMN active, - DROP COLUMN instance_id; - - -CREATE TABLE common_configs -( - id VARCHAR(255) NOT NULL PRIMARY KEY, - name VARCHAR(255) NOT NULL DEFAULT 'unnamed', - description TEXT, - created_at timestamp NOT NULL DEFAULT (now() at time zone 'utc'), - updated_at timestamp NOT NULL DEFAULT (now() at time zone 'utc'), - data jsonb NOT NULL DEFAULT '{}' -); - - --- +goose Down - -DROP TABLE common_configs; - diff --git a/migrations/00008_2020-06-03_create_version_config_table.sql b/migrations/00008_2020-06-03_create_version_config_table.sql deleted file mode 100644 index a83dd26..0000000 --- a/migrations/00008_2020-06-03_create_version_config_table.sql +++ /dev/null @@ -1,13 +0,0 @@ --- +goose Up -CREATE TABLE version_config -( - id VARCHAR(255) NOT NULL PRIMARY KEY, - config_id VARCHAR(255) NOT NULL, - config_version INT4 NOT NULL, - data JSONB NOT NULL DEFAULT '{}', - created_at TIMESTAMP NOT NULL DEFAULT (now() at time zone 'utc') -); - --- +goose Down -DROP TABLE version_config; - diff --git a/migrations/00009_2020-07-15_add_column_created_at.sql b/migrations/00009_2020-07-15_add_column_created_at.sql deleted file mode 100644 index f5ee012..0000000 --- a/migrations/00009_2020-07-15_add_column_created_at.sql +++ /dev/null @@ -1,7 +0,0 @@ --- +goose Up -ALTER TABLE version_config - ADD COLUMN IF NOT EXISTS created_at TIMESTAMP NOT NULL DEFAULT (now() at time zone 'utc'); - --- +goose Down - - diff --git a/migrations/20240503120009_init.sql b/migrations/20240503120009_init.sql new file mode 100644 index 0000000..58befe1 --- /dev/null +++ b/migrations/20240503120009_init.sql @@ -0,0 +1,55 @@ +-- +goose Up +-- +goose NO TRANSACTION +create table isp_config_service__module +( + id text not null primary key, + name text not null unique, + last_connected_at text, + last_disconnected_at text, + created_at text not null default (datetime('now')) +); + +create table isp_config_service__config +( + id text not null primary key, + name text not null, + module_id text not null, + data blob not null, + version int not null default 1, + active int not null default 0, + created_at text not null default (datetime('now')), + updated_at text not null default (datetime('now')), + foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade +); + +create index IX_isp_config_service__config__module_id on isp_config_service__config (module_id); + +create table isp_config_service__config_history +( + id text not null primary key, + config_id text not null, + data blob not null, + version int not null default 1, + admin_id int not null, + created_at text not null default (datetime('now')), + foreign key (config_id) references isp_config_service__config (id) on delete cascade on update cascade +); + +create index IX_isp_config_service__config_history__config_id on isp_config_service__config_history (config_id); + +create table isp_config_service__config_schema +( + id text not null primary key, + name text not null, + module_id text not null, + data blob not null, + version text not null default 1, + created_at text not null default (datetime('now')), + updated_at text not null default (datetime('now')), + foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade +); + +create index IX_isp_config_service__config_schema__module_id on isp_config_service__config_schema (module_id); + +-- +goose Down +drop table isp_config_service__module; diff --git a/model/client.go b/model/client.go deleted file mode 100644 index 0812884..0000000 --- a/model/client.go +++ /dev/null @@ -1,23 +0,0 @@ -package model - -import ( - "github.com/integration-system/isp-lib/v2/database" - log "github.com/integration-system/isp-log" - "github.com/integration-system/isp-log/stdcodes" -) - -var ( - DbClient = database.NewRxDbClient( - database.WithSchemaEnsuring(), - database.WithSchemaAutoInjecting(), - database.WithMigrationsEnsuring(), - database.WithInitializingErrorHandler(func(err *database.ErrorEvent) { - log.Fatal(stdcodes.InitializingDbError, err.Error()) - }), - ) - ConfigRep ConfigRepository = &configRepPg{rxClient: DbClient} - CommonConfigRep CommonConfigRepository = &commonConfigRepPg{rxClient: DbClient} - SchemaRep SchemaRepository = &schemaRepPg{rxClient: DbClient} - ModuleRep ModulesRepository = &modulesRepPg{rxClient: DbClient} - VersionStoreRep VersionConfigRepository = &versionConfigRepPg{rxClient: DbClient} -) diff --git a/model/common_config.go b/model/common_config.go deleted file mode 100644 index f1fcca0..0000000 --- a/model/common_config.go +++ /dev/null @@ -1,55 +0,0 @@ -//nolint: dupl -package model - -import ( - "github.com/go-pg/pg/v9" - "github.com/integration-system/isp-lib/v2/database" - "isp-config-service/entity" -) - -type CommonConfigRepository interface { - Snapshot() ([]entity.CommonConfig, error) - Upsert(config entity.CommonConfig) (*entity.CommonConfig, error) - Delete(identities []string) (int, error) -} - -type commonConfigRepPg struct { - rxClient *database.RxDbClient -} - -func (r *commonConfigRepPg) Snapshot() ([]entity.CommonConfig, error) { - configs := make([]entity.CommonConfig, 0) - err := r.rxClient.Visit(func(db *pg.DB) error { - return db.Model(&configs).Select() - }) - return configs, err -} - -func (r *commonConfigRepPg) Upsert(config entity.CommonConfig) (*entity.CommonConfig, error) { - err := r.rxClient.Visit(func(db *pg.DB) error { - _, err := db.Model(&config). - OnConflict("(id) DO UPDATE"). - Returning("*"). - Insert() - return err - }) - if err != nil { - return nil, err - } - return &config, err -} - -func (r *commonConfigRepPg) Delete(identities []string) (int, error) { - var err error - var res pg.Result - err = r.rxClient.Visit(func(db *pg.DB) error { - res, err = db.Model(&entity.CommonConfig{}). - Where("id IN (?)", pg.In(identities)). - Delete() - return err - }) - if err != nil { - return 0, err - } - return res.RowsAffected(), err -} diff --git a/model/config.go b/model/config.go deleted file mode 100644 index cf5fad1..0000000 --- a/model/config.go +++ /dev/null @@ -1,55 +0,0 @@ -//nolint:dupl -package model - -import ( - "github.com/go-pg/pg/v9" - "github.com/integration-system/isp-lib/v2/database" - "isp-config-service/entity" -) - -type ConfigRepository interface { - Snapshot() ([]entity.Config, error) - Upsert(config entity.Config) (*entity.Config, error) - Delete(identities []string) (int, error) -} - -type configRepPg struct { - rxClient *database.RxDbClient -} - -func (r *configRepPg) Snapshot() ([]entity.Config, error) { - configs := make([]entity.Config, 0) - err := r.rxClient.Visit(func(db *pg.DB) error { - return db.Model(&configs).Select() - }) - return configs, err -} - -func (r *configRepPg) Upsert(config entity.Config) (*entity.Config, error) { - err := r.rxClient.Visit(func(db *pg.DB) error { - _, err := db.Model(&config). - OnConflict("(id) DO UPDATE"). - Returning("*"). - Insert() - return err - }) - if err != nil { - return nil, err - } - return &config, err -} - -func (r *configRepPg) Delete(identities []string) (int, error) { - var err error - var res pg.Result - err = r.rxClient.Visit(func(db *pg.DB) error { - res, err = db.Model(&entity.Config{}). - Where("id IN (?)", pg.In(identities)). - Delete() - return err - }) - if err != nil { - return 0, err - } - return res.RowsAffected(), err -} diff --git a/model/module.go b/model/module.go deleted file mode 100644 index 883af9b..0000000 --- a/model/module.go +++ /dev/null @@ -1,54 +0,0 @@ -//nolint:dupl -package model - -import ( - "github.com/go-pg/pg/v9" - "github.com/integration-system/isp-lib/v2/database" - "isp-config-service/entity" -) - -type ModulesRepository interface { - Snapshot() ([]entity.Module, error) - Upsert(module entity.Module) (*entity.Module, error) - Delete(identities []string) (int, error) -} - -type modulesRepPg struct { - rxClient *database.RxDbClient -} - -func (r *modulesRepPg) Snapshot() ([]entity.Module, error) { - modules := make([]entity.Module, 0) - err := r.rxClient.Visit(func(db *pg.DB) error { - return db.Model(&modules).Select() - }) - return modules, err -} - -func (r *modulesRepPg) Upsert(module entity.Module) (*entity.Module, error) { - err := r.rxClient.Visit(func(db *pg.DB) error { - _, err := db.Model(&module). - OnConflict("(id) DO UPDATE"). - Returning("*"). - Insert() - return err - }) - if err != nil { - return nil, err - } - return &module, err -} - -func (r *modulesRepPg) Delete(identities []string) (int, error) { - var err error - var res pg.Result - err = r.rxClient.Visit(func(db *pg.DB) error { - res, err = db.Model(&entity.Module{}). - Where("id IN (?)", pg.In(identities)).Delete() - return err - }) - if err != nil { - return 0, err - } - return res.RowsAffected(), err -} diff --git a/model/schema.go b/model/schema.go deleted file mode 100644 index 5f3967a..0000000 --- a/model/schema.go +++ /dev/null @@ -1,54 +0,0 @@ -//nolint:dupl -package model - -import ( - "github.com/go-pg/pg/v9" - "github.com/integration-system/isp-lib/v2/database" - "isp-config-service/entity" -) - -type SchemaRepository interface { - Snapshot() ([]entity.ConfigSchema, error) - Upsert(schema entity.ConfigSchema) (*entity.ConfigSchema, error) - Delete(identities []string) (int, error) -} - -type schemaRepPg struct { - rxClient *database.RxDbClient -} - -func (r *schemaRepPg) Snapshot() ([]entity.ConfigSchema, error) { - schemas := make([]entity.ConfigSchema, 0) - err := r.rxClient.Visit(func(db *pg.DB) error { - return db.Model(&schemas).Select() - }) - return schemas, err -} - -func (r *schemaRepPg) Upsert(schema entity.ConfigSchema) (*entity.ConfigSchema, error) { - err := r.rxClient.Visit(func(db *pg.DB) error { - _, err := db.Model(&schema). - OnConflict("(id) DO UPDATE"). - Returning("*"). - Insert() - return err - }) - if err != nil { - return nil, err - } - return &schema, err -} - -func (r *schemaRepPg) Delete(identities []string) (int, error) { - var err error - var res pg.Result - err = r.rxClient.Visit(func(db *pg.DB) error { - res, err = db.Model(&entity.ConfigSchema{}). - Where("id IN (?)", pg.In(identities)).Delete() - return err - }) - if err != nil { - return 0, err - } - return res.RowsAffected(), err -} diff --git a/model/version_config.go b/model/version_config.go deleted file mode 100644 index 3aac07a..0000000 --- a/model/version_config.go +++ /dev/null @@ -1,53 +0,0 @@ -package model - -import ( - "github.com/go-pg/pg/v9" - "github.com/integration-system/isp-lib/v2/database" - "isp-config-service/entity" -) - -type VersionConfigRepository interface { - Snapshot() ([]entity.VersionConfig, error) - Upsert(model entity.VersionConfig) (*entity.VersionConfig, error) - Delete(identities string) (int, error) -} - -type versionConfigRepPg struct { - rxClient *database.RxDbClient -} - -func (r *versionConfigRepPg) Snapshot() ([]entity.VersionConfig, error) { - modules := make([]entity.VersionConfig, 0) - err := r.rxClient.Visit(func(db *pg.DB) error { - return db.Model(&modules).Select() - }) - return modules, err -} - -func (r *versionConfigRepPg) Upsert(module entity.VersionConfig) (*entity.VersionConfig, error) { - err := r.rxClient.Visit(func(db *pg.DB) error { - _, err := db.Model(&module). - OnConflict("(id) DO UPDATE"). - Returning("*"). - Insert() - return err - }) - if err != nil { - return nil, err - } - return &module, err -} - -func (r *versionConfigRepPg) Delete(id string) (int, error) { - var err error - var res pg.Result - err = r.rxClient.Visit(func(db *pg.DB) error { - res, err = db.Model(&entity.VersionConfig{}). - Where("id = ?", id).Delete() - return err - }) - if err != nil { - return 0, err - } - return res.RowsAffected(), err -} diff --git a/raft/manager.go b/raft/manager.go deleted file mode 100644 index 5d280a3..0000000 --- a/raft/manager.go +++ /dev/null @@ -1,175 +0,0 @@ -package raft - -import ( - "net" - "os" - "path/filepath" - "time" - - "github.com/hashicorp/raft" - "github.com/hashicorp/raft-boltdb/v2" - "github.com/integration-system/isp-log/adapter" - "github.com/pkg/errors" - "isp-config-service/codes" - "isp-config-service/conf" -) - -const ( - defaultSyncTimeout = 3 * time.Second - leaderNotificationChBuffer = 10 - dbFile = "raft_db" -) - -type ChangeLeaderNotification struct { - CurrentLeaderAddress string - IsLeader bool - LeaderElected bool -} - -type Raft struct { - r *raft.Raft - cfg conf.ClusterConfiguration - leaderObs *raft.Observer - leaderObsCh chan raft.Observation - changeLeaderCh chan ChangeLeaderNotification - closer chan struct{} -} - -func (r *Raft) BootstrapCluster() error { - peers := makeConfiguration(r.cfg.Peers) - - if f := r.r.BootstrapCluster(peers); f.Error() != nil { - return errors.WithMessage(f.Error(), "bootstrap cluster") - } - return nil -} - -func (r *Raft) SyncApply(command []byte) (interface{}, error) { - f := r.r.Apply(command, defaultSyncTimeout) - if err := f.Error(); err != nil { - return nil, err - } - return f.Response(), nil -} - -func (r *Raft) LeaderCh() <-chan ChangeLeaderNotification { - return r.changeLeaderCh -} - -func (r *Raft) GracefulShutdown() error { - r.r.DeregisterObserver(r.leaderObs) - close(r.closer) - close(r.leaderObsCh) - return r.r.Shutdown().Error() -} - -func (r *Raft) listenLeader() { - defer close(r.changeLeaderCh) - - for { - select { - case _, ok := <-r.leaderObsCh: - if !ok { - return - } - currentLeader := r.r.Leader() - select { - case r.changeLeaderCh <- ChangeLeaderNotification{ - IsLeader: r.r.State() == raft.Leader, - CurrentLeaderAddress: string(currentLeader), - LeaderElected: currentLeader != "", - }: - case <-r.closer: - return - default: - continue - } - case <-r.closer: - return - } - } -} - -func NewRaft(tcpListener net.Listener, configuration conf.ClusterConfiguration, state raft.FSM) (*Raft, error) { - logStore, store, snapshotStore, err := makeStores(configuration) - if err != nil { - return nil, err - } - - netLogger := adapter.NewHcLogger("RAFT-NET", codes.RaftLoggerCode) - streamLayer := &StreamLayer{Listener: tcpListener} - timeout := time.Duration(configuration.ConnectTimeoutSeconds) * time.Second - config := &raft.NetworkTransportConfig{ - Stream: streamLayer, - MaxPool: len(configuration.Peers), - Timeout: timeout, - Logger: netLogger, - ServerAddressProvider: transparentAddressProvider{}, - } - trans := raft.NewNetworkTransportWithConfig(config) - - cfg := raft.DefaultConfig() - cfg.Logger = adapter.NewHcLogger("RAFT", codes.RaftLoggerCode) - cfg.NoSnapshotRestoreOnStart = true - cfg.LocalID = raft.ServerID(configuration.OuterAddress) - r, err := raft.NewRaft(cfg, state, logStore, store, snapshotStore, trans) - if err != nil { - return nil, errors.WithMessage(err, "create raft") - } - - leaderObs, leaderObsCh := makeLeaderObserver() - r.RegisterObserver(leaderObs) - raft := &Raft{ - r: r, - cfg: configuration, - leaderObs: leaderObs, - leaderObsCh: leaderObsCh, - changeLeaderCh: make(chan ChangeLeaderNotification, leaderNotificationChBuffer), - closer: make(chan struct{}), - } - go raft.listenLeader() - - return raft, nil -} - -func makeLeaderObserver() (*raft.Observer, chan raft.Observation) { - ch := make(chan raft.Observation) - obs := raft.NewObserver(ch, false, func(o *raft.Observation) bool { - _, ok := o.Data.(raft.LeaderObservation) - return ok - }) - return obs, ch -} - -func makeConfiguration(peers []string) raft.Configuration { - servers := make([]raft.Server, len(peers)) - for i, peer := range peers { - servers[i] = raft.Server{ - ID: raft.ServerID(peer), - Address: raft.ServerAddress(peer), - } - } - return raft.Configuration{ - Servers: servers, - } -} - -func makeStores(configuration conf.ClusterConfiguration) (raft.LogStore, raft.StableStore, raft.SnapshotStore, error) { - if configuration.InMemory { - dbStore := raft.NewInmemStore() - snapStore := raft.NewInmemSnapshotStore() - return dbStore, dbStore, snapStore, nil - } else { - dbStore, err := raftboltdb.NewBoltStore(filepath.Join(configuration.DataDir, dbFile)) - if err != nil { - return nil, nil, nil, errors.WithMessage(err, "create db") - } - - snapStore, err := raft.NewFileSnapshotStore(configuration.DataDir, 1, os.Stdout) - if err != nil { - return nil, nil, nil, errors.WithMessage(err, "create snapshot store") - } - - return dbStore, dbStore, snapStore, nil - } -} diff --git a/raft/network.go b/raft/network.go deleted file mode 100644 index c7e87e4..0000000 --- a/raft/network.go +++ /dev/null @@ -1,22 +0,0 @@ -package raft - -import ( - "github.com/hashicorp/raft" - "net" - "time" -) - -type StreamLayer struct { - net.Listener -} - -func (r *StreamLayer) Dial(address raft.ServerAddress, timeout time.Duration) (net.Conn, error) { - return net.DialTimeout("tcp", string(address), timeout) -} - -type transparentAddressProvider struct { -} - -func (t transparentAddressProvider) ServerAddr(id raft.ServerID) (raft.ServerAddress, error) { - return raft.ServerAddress(id), nil -} diff --git a/repository/module.go b/repository/module.go new file mode 100644 index 0000000..22a1f57 --- /dev/null +++ b/repository/module.go @@ -0,0 +1,22 @@ +package repository + +import ( + "context" + + "github.com/txix-open/isp-kit/db" + "isp-config-service/entity" +) + +type Module struct { + db db.DB +} + +func NewModule(db db.DB) Module { + return Module{ + db: db, + } +} + +func (r Module) Upsert(ctx context.Context, module entity.Module) error { + +} diff --git a/routes/routes.go b/routes/routes.go new file mode 100644 index 0000000..88e309d --- /dev/null +++ b/routes/routes.go @@ -0,0 +1,46 @@ +package routes + +import ( + "net/http" + + "github.com/txix-open/etp/v3" + "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/grpc" + "github.com/txix-open/isp-kit/grpc/endpoint" + "isp-config-service/controller" +) + +type Controllers struct { + Module controller.Module +} + +func EndpointDescriptors() []cluster.EndpointDescriptor { + return endpointDescriptors(Controllers{}) +} + +func GrpcHandler(wrapper endpoint.Wrapper, c Controllers) *grpc.Mux { + muxer := grpc.NewMux() + for _, descriptor := range endpointDescriptors(c) { + muxer.Handle(descriptor.Path, wrapper.Endpoint(descriptor.Handler)) + } + return muxer +} + +func BindEtp(etpSrv *etp.Server, c Controllers) { + etpSrv.OnConnect(c.Module.OnConnect) + etpSrv.OnDisconnect(c.Module.OnDisconnect) + etpSrv.OnError(c.Module.OnError) + etpSrv.On(cluster.ModuleSendConfigSchema, etp.HandlerFunc(c.Module.OnModuleConfigSchema)) + etpSrv.On(cluster.ModuleSendRequirements, etp.HandlerFunc(c.Module.OnModuleRequirements)) + etpSrv.On(cluster.ModuleReady, etp.HandlerFunc(c.Module.OnModuleReady)) +} + +func HttpHandler(etpSrv *etp.Server) http.Handler { + httpMux := http.NewServeMux() + httpMux.Handle("/isp-etp/", etpSrv) + return httpMux +} + +func endpointDescriptors(c Controllers) []cluster.EndpointDescriptor { + return []cluster.EndpointDescriptor{} +} diff --git a/service/cluster_mesh.go b/service/cluster_mesh.go deleted file mode 100644 index 1aae2e9..0000000 --- a/service/cluster_mesh.go +++ /dev/null @@ -1,28 +0,0 @@ -package service - -import ( - "github.com/integration-system/isp-lib/v2/structure" - "isp-config-service/store/state" -) - -var ( - ClusterMesh clusterMeshService -) - -type clusterMeshService struct{} - -func (clusterMeshService) HandleUpdateBackendDeclarationCommand(declaration structure.BackendDeclaration, state state.WritableState) { - changed := state.WritableMesh().UpsertBackend(declaration) - if changed { - Discovery.BroadcastModuleAddresses(declaration.ModuleName, state.Mesh()) - Routes.BroadcastRoutes(state.Mesh()) - } -} - -func (clusterMeshService) HandleDeleteBackendDeclarationCommand(declaration structure.BackendDeclaration, state state.WritableState) { - deleted := state.WritableMesh().DeleteBackend(declaration) - if deleted { - Discovery.BroadcastModuleAddresses(declaration.ModuleName, state.Mesh()) - Routes.BroadcastRoutes(state.Mesh()) - } -} diff --git a/service/common_config.go b/service/common_config.go deleted file mode 100644 index e1f91a8..0000000 --- a/service/common_config.go +++ /dev/null @@ -1,91 +0,0 @@ -package service - -import ( - log "github.com/integration-system/isp-log" - codes2 "google.golang.org/grpc/codes" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/domain" - "isp-config-service/holder" - "isp-config-service/model" - "isp-config-service/store/state" -) - -var ( - CommonConfig = commonConfigService{} -) - -type commonConfigService struct{} - -func (s commonConfigService) HandleDeleteConfigsCommand( - deleteCommonConfig cluster.DeleteCommonConfig, state state.WritableState) cluster.ResponseWithError { - links := s.GetCommonConfigLinks(deleteCommonConfig.Id, state) - if len(links) > 0 { - return cluster.NewResponse(domain.DeleteCommonConfigResponse{Deleted: false, Links: links}) - } - ids := []string{deleteCommonConfig.Id} - deleted := state.WritableCommonConfigs().DeleteByIds(ids) - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - _, err := model.CommonConfigRep.Delete(ids) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "configIds": ids, - }).Errorf(codes.DatabaseOperationError, "delete configs: %v", err) - } - } - return cluster.NewResponse(domain.DeleteCommonConfigResponse{Deleted: deleted > 0}) -} - -func (s commonConfigService) HandleUpsertConfigCommand( - upsertConfig cluster.UpsertCommonConfig, state state.WritableState) cluster.ResponseWithError { - config := upsertConfig.Config - configsByName := state.CommonConfigs().GetByName(config.Name) - if upsertConfig.Create { - if len(configsByName) > 0 { - return cluster.NewResponseErrorf(codes2.AlreadyExists, - "common config with name %s already exists", upsertConfig.Config.Name) - } - config = state.WritableCommonConfigs().Create(config) - } else { - // Update - configs := state.CommonConfigs().GetByIds([]string{config.Id}) - if len(configs) == 0 { - return cluster.NewResponseErrorf(codes2.NotFound, "common config with id %s not found", config.Id) - } - if len(configsByName) > 0 && configs[0].Id != configsByName[0].Id { - return cluster.NewResponseErrorf(codes2.AlreadyExists, - "common config with name %s already exists", upsertConfig.Config.Name) - } - config.CreatedAt = configs[0].CreatedAt - state.WritableCommonConfigs().UpdateById(config) - } - - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - _, err := model.CommonConfigRep.Upsert(config) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "config": config, - }).Errorf(codes.DatabaseOperationError, "upsert common config: %v", err) - } - } - if !upsertConfig.Create { - configsToBroadcast := state.Configs().FilterByCommonConfigs([]string{config.Id}) - ConfigService.BroadcastActiveConfigs(state, configsToBroadcast...) - } - return cluster.NewResponse(config) -} - -func (s commonConfigService) GetCommonConfigLinks(commonConfigID string, state state.ReadonlyState) domain.CommonConfigLinks { - configs := state.Configs().FilterByCommonConfigs([]string{commonConfigID}) - result := make(domain.CommonConfigLinks) - for i := range configs { - module := state.Modules().GetById(configs[i].ModuleId) - if module != nil { - result[module.Name] = append(result[module.Name], configs[i].Name) - } - } - - return result -} diff --git a/service/config.go b/service/config.go deleted file mode 100644 index 893d21d..0000000 --- a/service/config.go +++ /dev/null @@ -1,233 +0,0 @@ -package service - -import ( - "encoding/json" - "fmt" - - "github.com/integration-system/bellows" - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - "github.com/pkg/errors" - "github.com/xeipuuv/gojsonschema" - codes2 "google.golang.org/grpc/codes" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/domain" - "isp-config-service/entity" - "isp-config-service/holder" - "isp-config-service/model" - "isp-config-service/store/state" -) - -var ( - ConfigService = configService{} -) - -type configService struct{} - -type validationSchemaError struct { - Description map[string]string -} - -func (e validationSchemaError) Error() string { - resp := "" - for field, desc := range e.Description { - resp = fmt.Sprintf("%s- %s%s\n", resp, field, desc) - } - return resp -} - -func (cs configService) GetCompiledConfig(moduleName string, state state.ReadonlyState) (map[string]interface{}, error) { - module := state.Modules().GetByName(moduleName) - if module == nil { - return nil, errors.Errorf("module with name %s not found", moduleName) - } - config := state.Configs().GetActiveByModuleId(module.Id) - if config == nil { - return nil, errors.Errorf("no active configs for moduleName %s, moduleId %s", moduleName, module.Id) - } - - return cs.CompileConfig(config.Data, state, config.CommonConfigs...), nil -} - -func (configService) CompileConfig( - data map[string]interface{}, state state.ReadonlyState, commonConfigsIds ...string) map[string]interface{} { - commonConfigs := state.CommonConfigs().GetByIds(commonConfigsIds) - configsToMerge := make([]map[string]interface{}, 0, len(commonConfigs)) - for _, common := range commonConfigs { - configsToMerge = append(configsToMerge, common.Data) - } - configsToMerge = append(configsToMerge, data) - - return mergeNestedMaps(configsToMerge...) -} - -func (cs configService) HandleActivateConfigCommand( - activateConfig cluster.ActivateConfig, state state.WritableState) cluster.ResponseWithError { - configs := state.Configs().GetByIds([]string{activateConfig.ConfigId}) - if len(configs) == 0 { - return cluster.NewResponseErrorf(codes2.NotFound, "config with id %s not found", activateConfig.ConfigId) - } - config := configs[0] - affected := state.WritableConfigs().Activate(config, activateConfig.Date) - if holder.ClusterClient.IsLeader() { - for i := range affected { - // TODO handle db errors - _, err := model.ConfigRep.Upsert(affected[i]) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "config": config, - }).Errorf(codes.DatabaseOperationError, "upsert config: %v", err) - } - } - } - config = affected[len(affected)-1] - cs.BroadcastActiveConfigs(state, config) - return cluster.NewResponse(config) -} - -func (configService) HandleDeleteConfigsCommand(deleteConfigs cluster.DeleteConfigs, state state.WritableState) int { - ids := deleteConfigs.Ids - deleted := state.WritableConfigs().DeleteByIds(ids) - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - _, err := model.ConfigRep.Delete(ids) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "configIds": ids, - }).Errorf(codes.DatabaseOperationError, "delete configs: %v", err) - } - } - return deleted -} - -func (cs configService) HandleUpsertConfigCommand(upsertConfig cluster.UpsertConfig, - state state.WritableState) cluster.ResponseWithError { - config := upsertConfig.Config - module := state.Modules().GetById(config.ModuleId) - if module == nil { - return cluster.NewResponseErrorf(codes2.NotFound, "moduleId %s not found", config.ModuleId) - } - - schemaStorage := state.Schemas().GetByModuleIds([]string{config.ModuleId}) - if len(schemaStorage) == 0 { - return cluster.NewResponseErrorf(codes2.NotFound, "schema for moduleId %s not found", config.ModuleId) - } - dataForValidate := cs.CompileConfig(config.Data, state, config.CommonConfigs...) - validSchema, err := cs.validateSchema(schemaStorage[0], dataForValidate) - - if !upsertConfig.Unsafe && err != nil { - switch err := err.(type) { - case validationSchemaError: - return cluster.NewResponse( - domain.CreateUpdateConfigResponse{ - Config: nil, - ErrorDetails: err.Description, - }) - default: - return cluster.NewResponseErrorf(codes2.Internal, "%v", err) - } - } - - if upsertConfig.Create { - config = state.WritableConfigs().Create(config) - } else { - // Update - configs := state.Configs().GetByIds([]string{config.Id}) - if len(configs) == 0 { - return cluster.NewResponseErrorf(codes2.NotFound, "config with id %s not found", config.Id) - } - oldCfg := configs[0] - config.CreatedAt = oldCfg.CreatedAt - config.Active = oldCfg.Active - config.Version = oldCfg.Version + 1 - state.WritableConfigs().UpdateById(config) - - ConfigHistory.SaveConfigVersion(upsertConfig.VersionId, oldCfg, state, upsertConfig.VersionCreateAt) - } - - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - _, err := model.ConfigRep.Upsert(config) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "config": config, - }).Errorf(codes.DatabaseOperationError, "upsert config: %v", err) - } - } - - cs.BroadcastActiveConfigs(state, config) - return cluster.NewResponse(domain.CreateUpdateConfigResponse{ - Config: &domain.ConfigModuleInfo{ - Config: config, - Valid: validSchema, - }, - ErrorDetails: nil, - }) -} - -func (cs configService) BroadcastActiveConfigs(state state.ReadonlyState, configs ...entity.Config) { - for i := range configs { - cfg := &configs[i] - if !cfg.Active { - continue - } - moduleID := cfg.ModuleId - module := state.Modules().GetById(moduleID) - moduleName := module.Name - room := Room.Module(moduleName) - - compiledConfig, err := cs.GetCompiledConfig(moduleName, state) - if err != nil { - go cs.broadcast(room, utils.ConfigError, []byte(err.Error())) - continue - } - data, err := json.Marshal(compiledConfig) - if err != nil { - go cs.broadcast(room, utils.ConfigError, []byte(err.Error())) - continue - } - go cs.broadcast(room, utils.ConfigSendConfigChanged, data) - } -} - -func (cs configService) broadcast(room, event string, body []byte) { - conns := holder.EtpServer.Rooms().ToBroadcast(room) - for _, conn := range conns { - err := EmitConnWithTimeout(conn, event, body) - if err != nil { - log.Errorf(codes.ConfigServiceBroadcastConfigError, "broadcast '%s' to %s: %v", event, conn.RemoteAddr(), err) - } - } -} - -func (configService) validateSchema(schema entity.ConfigSchema, data map[string]interface{}) (bool, error) { - schemaLoader := gojsonschema.NewGoLoader(schema.Schema) - documentLoader := gojsonschema.NewGoLoader(data) - result, err := gojsonschema.Validate(schemaLoader, documentLoader) - if err != nil { - return false, err - } else if result.Valid() { - return true, nil - } - - desc := make(map[string]string) - for _, value := range result.Errors() { - desc[value.Field()] = value.Description() - } - return false, validationSchemaError{Description: desc} -} - -func mergeNestedMaps(maps ...map[string]interface{}) map[string]interface{} { - if len(maps) == 1 { - return maps[0] - } - result := bellows.Flatten(maps[0]) - for i := 1; i < len(maps); i++ { - newFlatten := bellows.Flatten(maps[i]) - for k, v := range newFlatten { - result[k] = v - } - } - return bellows.Expand(result).(map[string]interface{}) -} diff --git a/service/discovery.go b/service/discovery.go deleted file mode 100644 index 295957e..0000000 --- a/service/discovery.go +++ /dev/null @@ -1,135 +0,0 @@ -package service - -import ( - "encoding/json" - "sync" - - etp "github.com/integration-system/isp-etp-go/v2" - "github.com/integration-system/isp-lib/v2/structure" - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/holder" - "isp-config-service/store/state" -) - -const ( - AllModules = "*" -) - -var ( - Discovery = &discoveryService{ - subs: make(map[string][]string), - } -) - -type discoveryService struct { - subs map[string][]string - lock sync.RWMutex -} - -func (ds *discoveryService) HandleDisconnect(connID string) { - ds.lock.Lock() - defer ds.lock.Unlock() - if rooms, ok := ds.subs[connID]; ok { - holder.EtpServer.Rooms().LeaveByConnId(connID, rooms...) - delete(ds.subs, connID) - } -} - -func (ds *discoveryService) Subscribe(conn etp.Conn, modules []string, mesh state.ReadonlyMesh) { - if len(modules) == 0 { - return - } - ds.lock.Lock() - defer ds.lock.Unlock() - eventsAddresses := make([][]structure.AddressConfiguration, 0, len(modules)) - rooms := make([]string, 0, len(modules)) - for _, module := range modules { - room := Room.AddressListener(module) - addressList := mesh.GetModuleAddresses(module) - eventsAddresses = append(eventsAddresses, addressList) - rooms = append(rooms, room) - } - ds.subs[conn.ID()] = rooms - holder.EtpServer.Rooms().Join(conn, rooms...) - - go func(events []string, eventsAddresses [][]structure.AddressConfiguration, conn etp.Conn) { - for i := range events { - event := events[i] - addressList := eventsAddresses[i] - body, err := json.Marshal(addressList) - if err != nil { - panic(err) - } - err = EmitConnWithTimeout(conn, event, body) - if err != nil { - log.Errorf(codes.DiscoveryServiceSendModulesError, "send module connected to %s: %v", conn.RemoteAddr(), err) - } - } - }(rooms, eventsAddresses, conn) -} - -func (ds *discoveryService) BroadcastModuleAddresses(moduleName string, mesh state.ReadonlyMesh) { - ds.lock.RLock() - defer ds.lock.RUnlock() - room := Room.AddressListener(moduleName) - event := utils.ModuleConnected(moduleName) - addressList := mesh.GetModuleAddresses(moduleName) - go ds.broadcastModuleAddrList(room, event, addressList) -} - -func (ds *discoveryService) BroadcastEvent(event cluster.BroadcastEvent) { - eventName := event.Event - payload := make([]byte, len(event.Payload)) - copy(payload, event.Payload) // TODO а тут точно нужно копирование ? - - if len(event.ModuleNames) == 1 && event.ModuleNames[0] == AllModules { - go func() { - conns := holder.EtpServer.Rooms().AllConns() - for _, conn := range conns { - err := EmitConnWithTimeout(conn, eventName, payload) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "eventName": eventName, - "remote_address": conn.RemoteAddr(), - }).Errorf(codes.DiscoveryServiceSendModulesError, "broadcast event to all modules: %v", err) - } - } - }() - return - } - - rooms := make([]string, 0, len(event.ModuleNames)) - for _, moduleName := range event.ModuleNames { - room := Room.Module(moduleName) - rooms = append(rooms, room) - } - go func(rooms []string) { - conns := holder.EtpServer.Rooms().ToBroadcast(rooms...) - for _, conn := range conns { - err := EmitConnWithTimeout(conn, eventName, payload) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "eventName": eventName, - "remote_address": conn.RemoteAddr(), - }).Errorf(codes.DiscoveryServiceSendModulesError, "broadcast event: %v", err) - } - } - }(rooms) -} - -func (ds *discoveryService) broadcastModuleAddrList(room string, event string, addressList []structure.AddressConfiguration) { - body, err := json.Marshal(addressList) - if err != nil { - panic(err) - } - conns := holder.EtpServer.Rooms().ToBroadcast(room) - for _, conn := range conns { - err := EmitConnWithTimeout(conn, event, body) - if err != nil { - log.Errorf(codes.DiscoveryServiceSendModulesError, "broadcast module connected to %s: %v", conn.RemoteAddr(), err) - } - } -} diff --git a/service/module.go b/service/module.go new file mode 100644 index 0000000..73c93da --- /dev/null +++ b/service/module.go @@ -0,0 +1,4 @@ +package service + +type Module struct { +} diff --git a/service/module_registry.go b/service/module_registry.go deleted file mode 100644 index c226c64..0000000 --- a/service/module_registry.go +++ /dev/null @@ -1,212 +0,0 @@ -package service - -import ( - "sort" - - "github.com/integration-system/isp-lib/v2/structure" - log "github.com/integration-system/isp-log" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/domain" - "isp-config-service/entity" - "isp-config-service/holder" - "isp-config-service/model" - "isp-config-service/store/state" -) - -var ( - ModuleRegistry = moduleRegistryService{} -) - -type moduleRegistryService struct{} - -func (s moduleRegistryService) HandleModuleConnectedCommand(module entity.Module, state state.WritableState) { - existedModule := state.WritableModules().GetByName(module.Name) - if existedModule == nil { - state.WritableModules().Create(module) - } else { - module.Id = existedModule.Id - module.CreatedAt = existedModule.CreatedAt - module.LastDisconnectedAt = existedModule.LastDisconnectedAt - state.WritableModules().UpdateByName(module) - } - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - _, err := model.ModuleRep.Upsert(module) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "module": module, - }).Errorf(codes.DatabaseOperationError, "upsert module: %v", err) - } - } -} - -func (moduleRegistryService) HandleModuleDisconnectedCommand(module entity.Module, state state.WritableState) { - existedModule := state.WritableModules().GetByName(module.Name) - if existedModule == nil { - state.WritableModules().Create(module) - } else { - module.Id = existedModule.Id - module.CreatedAt = existedModule.CreatedAt - module.LastConnectedAt = existedModule.LastConnectedAt - state.WritableModules().UpdateByName(module) - } - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - _, err := model.ModuleRep.Upsert(module) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "module": module, - }).Errorf(codes.DatabaseOperationError, "upsert module: %v", err) - } - } -} - -func (moduleRegistryService) HandleDeleteModulesCommand(deleteModules cluster.DeleteModules, state state.WritableState) int { - ids := deleteModules.Ids - deletedModules := state.WritableModules().DeleteByIds(ids) - - configsToDelete := state.WritableConfigs().GetByModuleIds(ids) - confIds := make([]string, 0, len(configsToDelete)) - for i := range configsToDelete { - confIds = append(confIds, configsToDelete[i].Id) - } - state.WritableConfigs().DeleteByIds(confIds) - - schemasToDelete := state.WritableSchemas().GetByModuleIds(ids) - schemaIds := make([]string, 0, len(schemasToDelete)) - for _, schema := range schemasToDelete { - schemaIds = append(schemaIds, schema.Id) - } - state.WritableSchemas().DeleteByIds(schemaIds) - - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - _, err := model.ModuleRep.Delete(ids) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "moduleIds": ids, - }).Errorf(codes.DatabaseOperationError, "delete modules: %v", err) - } - _, err = model.ConfigRep.Delete(confIds) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "configIds": confIds, - }).Errorf(codes.DatabaseOperationError, "delete configs: %v", err) - } - _, err = model.SchemaRep.Delete(schemaIds) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "schemaIds": schemaIds, - }).Errorf(codes.DatabaseOperationError, "delete schemas: %v", err) - } - } - return len(deletedModules) -} - -func (moduleRegistryService) ValidateConfig(config entity.Config, state state.ReadonlyState) bool { - schemas := state.Schemas().GetByModuleIds([]string{config.ModuleId}) - valid := false - - for _, s := range schemas { - dataForValidate := ConfigService.CompileConfig(config.Data, state, config.CommonConfigs...) - valid, _ = ConfigService.validateSchema(s, dataForValidate) - } - - return valid -} - -//nolint:funlen -func (moduleRegistryService) GetAggregatedModuleInfo(state state.ReadonlyState) []domain.ModuleInfo { - modules := state.Modules().GetAll() - modulesLen := len(modules) - result := make([]domain.ModuleInfo, 0, modulesLen) - resMap := make(map[string]domain.ModuleInfo, modulesLen) - nameIdMap := make(map[string]string) - idList := make([]string, modulesLen) - - for i, module := range modules { - idList[i] = module.Id - info := domain.ModuleInfo{ - Id: module.Id, - Name: module.Name, - CreatedAt: module.CreatedAt, - LastConnectedAt: module.LastConnectedAt, - LastDisconnectedAt: module.LastDisconnectedAt, - } - resMap[module.Id] = info - nameIdMap[module.Name] = module.Id - } - - configs := state.Configs().GetByModuleIds(idList) - for i := range configs { - info := resMap[configs[i].ModuleId] - info.Configs = append(info.Configs, domain.ConfigModuleInfo{ - Config: configs[i], - Valid: false, - }) - resMap[configs[i].ModuleId] = info - } - - schemas := state.Schemas().GetByModuleIds(idList) - for _, s := range schemas { - info := resMap[s.ModuleId] - schema := s.Schema - info.ConfigSchema = &schema - - for i := range info.Configs { - dataForValidate := ConfigService.CompileConfig(info.Configs[i].Data, state, info.Configs[i].CommonConfigs...) - info.Configs[i].Valid, _ = ConfigService.validateSchema(s, dataForValidate) - } - - resMap[s.ModuleId] = info - } - - for key := range resMap { - info := resMap[key] - backends := state.Mesh().GetBackends(info.Name) - conns := make([]domain.Connection, 0, len(backends)) - - if len(backends) > 0 { - back := backends[0] - requiredModules := make([]domain.ModuleDependency, 0, len(back.RequiredModules)) - for _, dep := range back.RequiredModules { - requiredModules = append(requiredModules, domain.ModuleDependency{ - Name: dep.Name, - Id: nameIdMap[dep.Name], - Required: dep.Required, - }) - } - info.RequiredModules = requiredModules - } - - for _, back := range backends { - con := domain.Connection{ - Version: back.Version, - LibVersion: back.LibVersion, - Address: back.Address, - Endpoints: make([]structure.EndpointDescriptor, len(back.Endpoints)), - } - copy(con.Endpoints, back.Endpoints) - sort.Slice(con.Endpoints, func(i, j int) bool { - return con.Endpoints[i].Path < con.Endpoints[j].Path - }) - con.EstablishedAt = info.LastConnectedAt - conns = append(conns, con) - } - sort.Slice(info.Configs, func(i, j int) bool { - return info.Configs[i].Version < info.Configs[j].Version - }) - sort.Slice(conns, func(i, j int) bool { - return conns[i].EstablishedAt.Before(conns[j].EstablishedAt) - }) - info.Status = conns - info.Active = len(conns) > 0 - result = append(result, info) - } - - sort.Slice(result, func(i, j int) bool { - return result[i].Name < result[j].Name - }) - return result -} diff --git a/service/room.go b/service/room.go deleted file mode 100644 index 1f51d04..0000000 --- a/service/room.go +++ /dev/null @@ -1,40 +0,0 @@ -package service - -import ( - "context" - "time" - - etp "github.com/integration-system/isp-etp-go/v2" - "github.com/integration-system/isp-lib/v2/utils" -) - -const ( - configWatchersRoomSuffix = "_config" - routesSubscribersRoom = "__routesSubscribers" -) - -const wsWriteTimeout = time.Second - -var ( - Room = roomService{} -) - -type roomService struct{} - -func (s roomService) Module(moduleName string) string { - return moduleName + configWatchersRoomSuffix -} - -func (s roomService) RoutesSubscribers() string { - return routesSubscribersRoom -} - -func (s roomService) AddressListener(moduleName string) string { - return utils.ModuleConnected(moduleName) -} - -func EmitConnWithTimeout(conn etp.Conn, event string, body []byte) error { - ctx, cancel := context.WithTimeout(context.Background(), wsWriteTimeout) - defer cancel() - return conn.Emit(ctx, event, body) -} diff --git a/service/routes.go b/service/routes.go deleted file mode 100644 index 1d4f812..0000000 --- a/service/routes.go +++ /dev/null @@ -1,54 +0,0 @@ -package service - -import ( - "encoding/json" - - etp "github.com/integration-system/isp-etp-go/v2" - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - "isp-config-service/codes" - "isp-config-service/holder" - "isp-config-service/store/state" -) - -var Routes routesService - -type routesService struct{} - -func (rs *routesService) HandleDisconnect(connID string) { - holder.EtpServer.Rooms().LeaveByConnId(connID, Room.RoutesSubscribers()) -} - -func (rs *routesService) SubscribeRoutes(conn etp.Conn, mesh state.ReadonlyMesh) { - holder.EtpServer.Rooms().Join(conn, Room.RoutesSubscribers()) - routes := mesh.GetRoutes() - body, err := json.Marshal(routes) - if err != nil { - panic(err) - } - - go func() { - err := EmitConnWithTimeout(conn, utils.ConfigSendRoutesWhenConnected, body) - if err != nil { - log.Errorf(codes.RoutesServiceSendRoutesError, "send routes to %s: %v", conn.RemoteAddr(), err) - } - }() -} - -func (rs *routesService) BroadcastRoutes(mesh state.ReadonlyMesh) { - routes := mesh.GetRoutes() - body, err := json.Marshal(routes) - if err != nil { - panic(err) - } - - go func() { - conns := holder.EtpServer.Rooms().ToBroadcast(Room.RoutesSubscribers()) - for _, conn := range conns { - err := EmitConnWithTimeout(conn, utils.ConfigSendRoutesChanged, body) - if err != nil { - log.Errorf(codes.RoutesServiceSendRoutesError, "broadcast routes to %s: %v", conn.RemoteAddr(), err) - } - } - }() -} diff --git a/service/rqlite/db/db.go b/service/rqlite/db/db.go new file mode 100644 index 0000000..9367e85 --- /dev/null +++ b/service/rqlite/db/db.go @@ -0,0 +1,120 @@ +package db + +import ( + "context" + "database/sql" + + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + "github.com/tidwall/gjson" + "github.com/txix-open/isp-kit/http/httpcli" + "github.com/txix-open/isp-kit/json" +) + +type DB struct { + cli *httpcli.Client +} + +func Open(ctx context.Context, dsn string, client *httpcli.Client) (*DB, error) { + client.GlobalRequestConfig().BaseUrl = dsn + db := &DB{ + cli: client, + } + + m := map[string]any{} + err := db.SelectRow(ctx, &m, `SELECT 1 as test`) + if err != nil { + return nil, errors.WithMessage(err, "ping db") + } + + return db, nil +} + +func (d DB) Select(ctx context.Context, ptr any, query string, args ...any) error { + result := &Result{ + Rows: ptr, + } + resp := Response{ + Results: []*Result{result}, + } + err := d.cli.Post("/db/query"). + QueryParams(map[string]any{ + "timings": true, + "associative": true, + }).JsonRequestBody(Request(query, args...)). + JsonResponseBody(&resp). + StatusCodeToError(). + DoWithoutResponse(ctx) + if err != nil { + return errors.WithMessage(err, "call rqlite") + } + if result.Error != "" { + return errors.Errorf("sqlite: %s", result.Error) + } + return nil +} + +func (d DB) SelectRow(ctx context.Context, ptr any, query string, args ...any) error { + result := &Result{} + resp := Response{ + Results: []*Result{result}, + } + httpResp, err := d.cli.Post("/db/query"). + QueryParams(map[string]any{ + "timings": true, + "associative": true, + }).JsonRequestBody(Request(query, args...)). + JsonResponseBody(&resp). + StatusCodeToError(). + Do(ctx) + if err != nil { + return errors.WithMessage(err, "call rqlite") + } + defer httpResp.Close() + + if result.Error != "" { + return errors.Errorf("sqlite: %s", result.Error) + } + + body, _ := httpResp.Body() + rows := gjson.GetBytes(body, "results.0.rows") + elems := rows.Array() + if rows.IsArray() && len(elems) == 0 { + return sql.ErrNoRows + } + err = json.Unmarshal([]byte(elems[0].Raw), ptr) + if err != nil { + return errors.WithMessage(err, "json unmarshal") + } + + return nil +} + +func (d DB) Exec(ctx context.Context, query string, args ...any) (sql.Result, error) { + result := &Result{} + resp := Response{ + Results: []*Result{result}, + } + err := d.cli.Post("/db/execute"). + QueryParams(map[string]any{ + "timings": true, + }).JsonRequestBody(Request(query, args...)). + JsonResponseBody(&resp). + StatusCodeToError(). + DoWithoutResponse(ctx) + if err != nil { + return nil, errors.WithMessage(err, "call rqlite") + } + if result.Error != "" { + return nil, errors.Errorf("sqlite: %s", result.Error) + } + return result, nil +} + +func (d DB) ExecNamed(ctx context.Context, query string, arg any) (sql.Result, error) { + query, args, err := sqlx.Named(query, arg) + if err != nil { + return nil, errors.WithMessage(err, "map to unnamed query") + } + return d.Exec(ctx, query, args...) +} diff --git a/service/rqlite/db/request.go b/service/rqlite/db/request.go new file mode 100644 index 0000000..75a4afa --- /dev/null +++ b/service/rqlite/db/request.go @@ -0,0 +1,5 @@ +package db + +func Request(query string, args ...any) []any { + return append([]any{query}, args...) +} diff --git a/service/rqlite/db/result.go b/service/rqlite/db/result.go new file mode 100644 index 0000000..0af161f --- /dev/null +++ b/service/rqlite/db/result.go @@ -0,0 +1,23 @@ +package db + +type Result struct { + LastInsertIdValue int64 `json:"last_insert_id"` + RowsAffectedValue int64 `json:"rows_affected"` + Types map[string]string `json:"types"` + Time float64 `json:"time"` + Rows any `json:"rows"` + Error string `json:"error"` +} + +func (r Result) LastInsertId() (int64, error) { + return r.LastInsertIdValue, nil +} + +func (r Result) RowsAffected() (int64, error) { + return r.RowsAffectedValue, nil +} + +type Response struct { + Results []*Result `json:"results"` + Time float64 `json:"time"` +} diff --git a/service/rqlite/flags.go b/service/rqlite/flags.go new file mode 100644 index 0000000..e38ca91 --- /dev/null +++ b/service/rqlite/flags.go @@ -0,0 +1,532 @@ +// nolint +package rqlite + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io" + "net" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "time" +) + +const ( + DiscoModeNone = "" + DiscoModeConsulKV = "consul-kv" + DiscoModeEtcdKV = "etcd-kv" + DiscoModeDNS = "dns" + DiscoModeDNSSRV = "dns-srv" + + HTTPAddrFlag = "http-addr" + HTTPAdvAddrFlag = "http-adv-addr" + RaftAddrFlag = "raft-addr" + RaftAdvAddrFlag = "raft-adv-addr" + + HTTPx509CertFlag = "http-cert" + HTTPx509KeyFlag = "http-key" + NodeX509CertFlag = "node-cert" + NodeX509KeyFlag = "node-key" +) + +// Config represents the configuration as set by command-line flags. +// All variables will be set, unless explicit noted. +type Config struct { + // DataPath is path to node data. Always set. + DataPath string + + // HTTPAddr is the bind network address for the HTTP Server. + // It never includes a trailing HTTP or HTTPS. + HTTPAddr string + + // HTTPAdv is the advertised HTTP server network. + HTTPAdv string + + // HTTPAllowOrigin is the value to set for Access-Control-Allow-Origin HTTP header. + HTTPAllowOrigin string + + // AuthFile is the path to the authentication file. May not be set. + AuthFile string `filepath:"true"` + + // AutoBackupFile is the path to the auto-backup file. May not be set. + AutoBackupFile string `filepath:"true"` + + // AutoRestoreFile is the path to the auto-restore file. May not be set. + AutoRestoreFile string `filepath:"true"` + + // HTTPx509CACert is the path to the CA certificate file for when this node verifies + // other certificates for any HTTP communications. May not be set. + HTTPx509CACert string `filepath:"true"` + + // HTTPx509Cert is the path to the X509 cert for the HTTP server. May not be set. + HTTPx509Cert string `filepath:"true"` + + // HTTPx509Key is the path to the private key for the HTTP server. May not be set. + HTTPx509Key string `filepath:"true"` + + // HTTPVerifyClient indicates whether the HTTP server should verify client certificates. + HTTPVerifyClient bool + + // NodeX509CACert is the path to the CA certificate file for when this node verifies + // other certificates for any inter-node communications. May not be set. + NodeX509CACert string `filepath:"true"` + + // NodeX509Cert is the path to the X509 cert for the Raft server. May not be set. + NodeX509Cert string `filepath:"true"` + + // NodeX509Key is the path to the X509 key for the Raft server. May not be set. + NodeX509Key string `filepath:"true"` + + // NoNodeVerify disables checking other nodes' Node X509 certs for validity. + NoNodeVerify bool + + // NodeVerifyClient enable mutual TLS for node-to-node communication. + NodeVerifyClient bool + + // NodeVerifyServerName is the hostname to verify on the certificates returned by nodes. + // If NoNodeVerify is true this field is ignored. + NodeVerifyServerName string + + // NodeID is the Raft ID for the node. + NodeID string + + // RaftAddr is the bind network address for the Raft server. + RaftAddr string + + // RaftAdv is the advertised Raft server address. + RaftAdv string + + // JoinAddrs is the list of Raft addresses to use for a join attempt. + JoinAddrs string + + // JoinAttempts is the number of times a node should attempt to join using a + // given address. + JoinAttempts int + + // JoinInterval is the time between retrying failed join operations. + JoinInterval time.Duration + + // JoinAs sets the user join attempts should be performed as. May not be set. + JoinAs string + + // BootstrapExpect is the minimum number of nodes required for a bootstrap. + BootstrapExpect int + + // BootstrapExpectTimeout is the maximum time a bootstrap operation can take. + BootstrapExpectTimeout time.Duration + + // DiscoMode sets the discovery mode. May not be set. + DiscoMode string + + // DiscoKey sets the discovery prefix key. + DiscoKey string + + // DiscoConfig sets the path to any discovery configuration file. May not be set. + DiscoConfig string + + // OnDiskPath sets the path to the SQLite file. May not be set. + OnDiskPath string + + // FKConstraints enables SQLite foreign key constraints. + FKConstraints bool + + // AutoVacInterval sets the automatic VACUUM interval. Use 0s to disable. + AutoVacInterval time.Duration + + // RaftLogLevel sets the minimum logging level for the Raft subsystem. + RaftLogLevel string + + // RaftNonVoter controls whether this node is a voting, read-only node. + RaftNonVoter bool + + // RaftSnapThreshold is the number of outstanding log entries that trigger snapshot. + RaftSnapThreshold uint64 + + // RaftSnapThreshold is the size of a SQLite WAL file which will trigger a snapshot. + RaftSnapThresholdWALSize uint64 + + // RaftSnapInterval sets the threshold check interval. + RaftSnapInterval time.Duration + + // RaftLeaderLeaseTimeout sets the leader lease timeout. + RaftLeaderLeaseTimeout time.Duration + + // RaftHeartbeatTimeout specifies the time in follower state without contact + // from a Leader before the node attempts an election. + RaftHeartbeatTimeout time.Duration + + // RaftElectionTimeout specifies the time in candidate state without contact + // from a Leader before the node attempts an election. + RaftElectionTimeout time.Duration + + // RaftApplyTimeout sets the Log-apply timeout. + RaftApplyTimeout time.Duration + + // RaftShutdownOnRemove sets whether Raft should be shutdown if the node is removed + RaftShutdownOnRemove bool + + // RaftClusterRemoveOnShutdown sets whether the node should remove itself from the cluster on shutdown + RaftClusterRemoveOnShutdown bool + + // RaftStepdownOnShutdown sets whether Leadership should be relinquished on shutdown + RaftStepdownOnShutdown bool + + // RaftReapNodeTimeout sets the duration after which a non-reachable voting node is + // reaped i.e. removed from the cluster. + RaftReapNodeTimeout time.Duration + + // RaftReapReadOnlyNodeTimeout sets the duration after which a non-reachable non-voting node is + // reaped i.e. removed from the cluster. + RaftReapReadOnlyNodeTimeout time.Duration + + // ClusterConnectTimeout sets the timeout when initially connecting to another node in + // the cluster, for non-Raft communications. + ClusterConnectTimeout time.Duration + + // WriteQueueCap is the default capacity of Execute queues + WriteQueueCap int + + // WriteQueueBatchSz is the default batch size for Execute queues + WriteQueueBatchSz int + + // WriteQueueTimeout is the default time after which any data will be sent on + // Execute queues, if a batch size has not been reached. + WriteQueueTimeout time.Duration + + // WriteQueueTx controls whether writes from the queue are done within a transaction. + WriteQueueTx bool + + // CPUProfile enables CPU profiling. + CPUProfile string + + // MemProfile enables memory profiling. + MemProfile string +} + +// Validate checks the configuration for internal consistency, and activates +// important rqlite policies. It must be called at least once on a Config +// object before the Config object is used. It is OK to call more than +// once. +func (c *Config) Validate() error { + dataPath, err := filepath.Abs(c.DataPath) + if err != nil { + return fmt.Errorf("failed to determine absolute data path: %s", err.Error()) + } + c.DataPath = dataPath + + err = c.CheckFilePaths() + if err != nil { + return err + } + + if !bothUnsetSet(c.HTTPx509Cert, c.HTTPx509Key) { + return fmt.Errorf("either both -%s and -%s must be set, or neither", HTTPx509CertFlag, HTTPx509KeyFlag) + } + if !bothUnsetSet(c.NodeX509Cert, c.NodeX509Key) { + return fmt.Errorf("either both -%s and -%s must be set, or neither", NodeX509CertFlag, NodeX509KeyFlag) + + } + + if c.RaftAddr == c.HTTPAddr { + return errors.New("HTTP and Raft addresses must differ") + } + + // Enforce policies regarding addresses + if c.RaftAdv == "" { + c.RaftAdv = c.RaftAddr + } + if c.HTTPAdv == "" { + c.HTTPAdv = c.HTTPAddr + } + + // Node ID policy + if c.NodeID == "" { + c.NodeID = c.RaftAdv + } + + // Perform some address validity checks. + if strings.HasPrefix(strings.ToLower(c.HTTPAddr), "http") || + strings.HasPrefix(strings.ToLower(c.HTTPAdv), "http") { + return errors.New("HTTP options should not include protocol (http:// or https://)") + } + if _, _, err := net.SplitHostPort(c.HTTPAddr); err != nil { + return errors.New("HTTP bind address not valid") + } + + hadv, _, err := net.SplitHostPort(c.HTTPAdv) + if err != nil { + return errors.New("HTTP advertised HTTP address not valid") + } + if addr := net.ParseIP(hadv); addr != nil && addr.IsUnspecified() { + return fmt.Errorf("advertised HTTP address is not routable (%s), specify it via -%s or -%s", + hadv, HTTPAddrFlag, HTTPAdvAddrFlag) + } + + if _, rp, err := net.SplitHostPort(c.RaftAddr); err != nil { + return errors.New("raft bind address not valid") + } else if _, err := strconv.Atoi(rp); err != nil { + return errors.New("raft bind port not valid") + } + + radv, rp, err := net.SplitHostPort(c.RaftAdv) + if err != nil { + return errors.New("raft advertised address not valid") + } + if addr := net.ParseIP(radv); addr != nil && addr.IsUnspecified() { + return fmt.Errorf("advertised Raft address is not routable (%s), specify it via -%s or -%s", + radv, RaftAddrFlag, RaftAdvAddrFlag) + } + if _, err := strconv.Atoi(rp); err != nil { + return errors.New("raft advertised port is not valid") + } + + if c.RaftAdv == c.HTTPAdv { + return errors.New("advertised HTTP and Raft addresses must differ") + } + + // Enforce bootstrapping policies + if c.BootstrapExpect > 0 && c.RaftNonVoter { + return errors.New("bootstrapping only applicable to voting nodes") + } + + // Join parameters OK? + if c.JoinAddrs != "" { + addrs := strings.Split(c.JoinAddrs, ",") + for i := range addrs { + if _, _, err := net.SplitHostPort(addrs[i]); err != nil { + return fmt.Errorf("%s is an invalid join address", addrs[i]) + } + + if c.BootstrapExpect == 0 { + if addrs[i] == c.RaftAdv || addrs[i] == c.RaftAddr { + return errors.New("node cannot join with itself unless bootstrapping") + } + if c.AutoRestoreFile != "" { + return errors.New("auto-restoring cannot be used when joining a cluster") + } + } + } + + if c.DiscoMode != "" { + return errors.New("disco mode cannot be used when also explicitly joining a cluster") + } + } + + // Valid disco mode? + switch c.DiscoMode { + case "": + case DiscoModeEtcdKV, DiscoModeConsulKV: + if c.BootstrapExpect > 0 { + return fmt.Errorf("bootstrapping not applicable when using %s", c.DiscoMode) + } + case DiscoModeDNS, DiscoModeDNSSRV: + if c.BootstrapExpect == 0 && !c.RaftNonVoter { + return fmt.Errorf("bootstrap-expect value required when using %s with a voting node", c.DiscoMode) + } + default: + return fmt.Errorf("disco mode must be one of %s, %s, %s, or %s", + DiscoModeConsulKV, DiscoModeEtcdKV, DiscoModeDNS, DiscoModeDNSSRV) + } + + return nil +} + +// JoinAddresses returns the join addresses set at the command line. Returns nil +// if no join addresses were set. +func (c *Config) JoinAddresses() []string { + if c.JoinAddrs == "" { + return nil + } + return strings.Split(c.JoinAddrs, ",") +} + +// HTTPURL returns the fully-formed, advertised HTTP API address for this config, including +// protocol, host and port. +func (c *Config) HTTPURL() string { + apiProto := "http" + if c.HTTPx509Cert != "" { + apiProto = "https" + } + return fmt.Sprintf("%s://%s", apiProto, c.HTTPAdv) +} + +// RaftPort returns the port on which the Raft system is listening. Validate must +// have been called before calling this method. +func (c *Config) RaftPort() int { + _, port, err := net.SplitHostPort(c.RaftAddr) + if err != nil { + panic("RaftAddr not valid") + } + p, err := strconv.Atoi(port) + if err != nil { + panic("RaftAddr port not valid") + } + return p +} + +// DiscoConfigReader returns a ReadCloser providing access to the Disco config. +// The caller must call close on the ReadCloser when finished with it. If no +// config was supplied, it returns nil. +func (c *Config) DiscoConfigReader() io.ReadCloser { + var rc io.ReadCloser + if c.DiscoConfig == "" { + return nil + } + + // Open config file. If opening fails, assume string is the literal config. + cfgFile, err := os.Open(c.DiscoConfig) + if err != nil { + rc = io.NopCloser(bytes.NewReader([]byte(c.DiscoConfig))) + } else { + rc = cfgFile + } + return rc +} + +// CheckFilePaths checks that all file paths in the config exist. +// Empty filepaths are ignored. +func (c *Config) CheckFilePaths() error { + v := reflect.ValueOf(c).Elem() + + // Iterate through the fields of the struct + for i := 0; i < v.NumField(); i++ { + field := v.Type().Field(i) + fieldValue := v.Field(i) + + if fieldValue.Kind() != reflect.String { + continue + } + + if tagValue, ok := field.Tag.Lookup("filepath"); ok && tagValue == "true" { + filePath := fieldValue.String() + if filePath == "" { + continue + } + _, err := os.Stat(filePath) + if os.IsNotExist(err) { + return fmt.Errorf("%s does not exist", filePath) + } + } + } + return nil +} + +// BuildInfo is build information for display at command line. +type BuildInfo struct { + Version string + Commit string + Branch string + SQLiteVersion string +} + +// ParseFlags parses the command line, and returns the configuration. +func ParseFlags(name, desc string, build *BuildInfo) (*Config, error) { + if flag.Parsed() { + return nil, fmt.Errorf("command-line flags already parsed") + } + config := &Config{} + showVersion := false + + fs := flag.NewFlagSet(name, flag.ExitOnError) + + fs.StringVar(&config.NodeID, "node-id", "", "Unique ID for node. If not set, set to advertised Raft address") + fs.StringVar(&config.HTTPAddr, HTTPAddrFlag, "localhost:4001", "HTTP server bind address. To enable HTTPS, set X.509 certificate and key") + fs.StringVar(&config.HTTPAdv, HTTPAdvAddrFlag, "", "Advertised HTTP address. If not set, same as HTTP server bind address") + fs.StringVar(&config.HTTPAllowOrigin, "http-allow-origin", "", "Value to set for Access-Control-Allow-Origin HTTP header") + fs.StringVar(&config.HTTPx509CACert, "http-ca-cert", "", "Path to X.509 CA certificate for HTTPS") + fs.StringVar(&config.HTTPx509Cert, HTTPx509CertFlag, "", "Path to HTTPS X.509 certificate") + fs.StringVar(&config.HTTPx509Key, HTTPx509KeyFlag, "", "Path to HTTPS X.509 private key") + fs.BoolVar(&config.HTTPVerifyClient, "http-verify-client", false, "Enable mutual TLS for HTTPS") + fs.StringVar(&config.NodeX509CACert, "node-ca-cert", "", "Path to X.509 CA certificate for node-to-node encryption") + fs.StringVar(&config.NodeX509Cert, NodeX509CertFlag, "", "Path to X.509 certificate for node-to-node mutual authentication and encryption") + fs.StringVar(&config.NodeX509Key, NodeX509KeyFlag, "", "Path to X.509 private key for node-to-node mutual authentication and encryption") + fs.BoolVar(&config.NoNodeVerify, "node-no-verify", false, "Skip verification of any node-node certificate") + fs.BoolVar(&config.NodeVerifyClient, "node-verify-client", false, "Enable mutual TLS for node-to-node communication") + fs.StringVar(&config.NodeVerifyServerName, "node-verify-server-name", "", "Hostname to verify on certificate returned by a node") + fs.StringVar(&config.AuthFile, "auth", "", "Path to authentication and authorization file. If not set, not enabled") + fs.StringVar(&config.AutoBackupFile, "auto-backup", "", "Path to automatic backup configuration file. If not set, not enabled") + fs.StringVar(&config.AutoRestoreFile, "auto-restore", "", "Path to automatic restore configuration file. If not set, not enabled") + fs.StringVar(&config.RaftAddr, RaftAddrFlag, "localhost:4002", "Raft communication bind address") + fs.StringVar(&config.RaftAdv, RaftAdvAddrFlag, "", "Advertised Raft communication address. If not set, same as Raft bind address") + fs.StringVar(&config.JoinAddrs, "join", "", "Comma-delimited list of nodes, in host:port form, through which a cluster can be joined") + fs.IntVar(&config.JoinAttempts, "join-attempts", 5, "Number of join attempts to make") + fs.DurationVar(&config.JoinInterval, "join-interval", 3*time.Second, "Period between join attempts") + fs.StringVar(&config.JoinAs, "join-as", "", "Username in authentication file to join as. If not set, joins anonymously") + fs.IntVar(&config.BootstrapExpect, "bootstrap-expect", 0, "Minimum number of nodes required for a bootstrap") + fs.DurationVar(&config.BootstrapExpectTimeout, "bootstrap-expect-timeout", 120*time.Second, "Maximum time for bootstrap process") + fs.StringVar(&config.DiscoMode, "disco-mode", "", "Choose clustering discovery mode. If not set, no node discovery is performed") + fs.StringVar(&config.DiscoKey, "disco-key", "rqlite", "Key prefix for cluster discovery service") + fs.StringVar(&config.DiscoConfig, "disco-config", "", "Set discovery config, or path to cluster discovery config file") + fs.StringVar(&config.OnDiskPath, "on-disk-path", "", "Path for SQLite on-disk database file. If not set, use a file in data directory") + fs.BoolVar(&config.FKConstraints, "fk", false, "Enable SQLite foreign key constraints") + fs.BoolVar(&showVersion, "version", false, "Show version information and exit") + fs.DurationVar(&config.AutoVacInterval, "auto-vacuum-int", 0, "Period between automatic VACUUMs. It not set, not enabled") + fs.BoolVar(&config.RaftNonVoter, "raft-non-voter", false, "Configure as non-voting node") + fs.DurationVar(&config.RaftHeartbeatTimeout, "raft-timeout", time.Second, "Raft heartbeat timeout") + fs.DurationVar(&config.RaftElectionTimeout, "raft-election-timeout", time.Second, "Raft election timeout") + fs.DurationVar(&config.RaftApplyTimeout, "raft-apply-timeout", 10*time.Second, "Raft apply timeout") + fs.Uint64Var(&config.RaftSnapThreshold, "raft-snap", 8192, "Number of outstanding log entries which triggers Raft snapshot") + fs.Uint64Var(&config.RaftSnapThresholdWALSize, "raft-snap-wal-size", 4*1024*1024, "SQLite WAL file size in bytes which triggers Raft snapshot. Set to 0 to disable") + fs.DurationVar(&config.RaftSnapInterval, "raft-snap-int", 10*time.Second, "Snapshot threshold check interval") + fs.DurationVar(&config.RaftLeaderLeaseTimeout, "raft-leader-lease-timeout", 0, "Raft leader lease timeout. Use 0s for Raft default") + fs.BoolVar(&config.RaftStepdownOnShutdown, "raft-shutdown-stepdown", true, "If leader, stepdown before shutting down. Enabled by default") + fs.BoolVar(&config.RaftShutdownOnRemove, "raft-remove-shutdown", false, "Shutdown Raft if node removed from cluster") + fs.BoolVar(&config.RaftClusterRemoveOnShutdown, "raft-cluster-remove-shutdown", false, "Node removes itself from cluster on graceful shutdown") + fs.StringVar(&config.RaftLogLevel, "raft-log-level", "WARN", "Minimum log level for Raft module") + fs.DurationVar(&config.RaftReapNodeTimeout, "raft-reap-node-timeout", 0*time.Hour, "Time after which a non-reachable voting node will be reaped. If not set, no reaping takes place") + fs.DurationVar(&config.RaftReapReadOnlyNodeTimeout, "raft-reap-read-only-node-timeout", 0*time.Hour, "Time after which a non-reachable non-voting node will be reaped. If not set, no reaping takes place") + fs.DurationVar(&config.ClusterConnectTimeout, "cluster-connect-timeout", 30*time.Second, "Timeout for initial connection to other nodes") + fs.IntVar(&config.WriteQueueCap, "write-queue-capacity", 1024, "QueuedWrites queue capacity") + fs.IntVar(&config.WriteQueueBatchSz, "write-queue-batch-size", 128, "QueuedWrites queue batch size") + fs.DurationVar(&config.WriteQueueTimeout, "write-queue-timeout", 50*time.Millisecond, "QueuedWrites queue timeout") + fs.BoolVar(&config.WriteQueueTx, "write-queue-tx", false, "Use a transaction when processing a queued write") + fs.StringVar(&config.CPUProfile, "cpu-profile", "", "Path to file for CPU profiling information") + fs.StringVar(&config.MemProfile, "mem-profile", "", "Path to file for memory profiling information") + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "\n%s\n\n", desc) + fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", name) + fs.PrintDefaults() + } + fs.Parse(os.Args[1:]) + + if showVersion { + msg := fmt.Sprintf("%s %s %s %s %s sqlite%s (commit %s, branch %s, compiler %s)", + name, build.Version, runtime.GOOS, runtime.GOARCH, runtime.Version(), build.SQLiteVersion, + build.Commit, build.Branch, runtime.Compiler) + errorExit(0, msg) + } + + // Ensure, if set explicitly, that reap times are not too low. + fs.Visit(func(f *flag.Flag) { + if f.Name == "raft-reap-node-timeout" || f.Name == "raft-reap-read-only-node-timeout" { + d, err := time.ParseDuration(f.Value.String()) + if err != nil { + errorExit(1, fmt.Sprintf("failed to parse duration: %s", err.Error())) + } + if d <= 0 { + errorExit(1, fmt.Sprintf("-%s must be greater than 0", f.Name)) + } + } + }) + + return config, nil +} + +func errorExit(code int, msg string) { + if code != 0 { + fmt.Fprintf(os.Stderr, "fatal: ") + } + fmt.Fprintf(os.Stderr, "%s\n", msg) + os.Exit(code) +} + +// bothUnsetSet returns true if both a and b are unset, or both are set. +func bothUnsetSet(a, b string) bool { + return (a == "" && b == "") || (a != "" && b != "") +} diff --git a/service/rqlite/goose_store/dialect.go b/service/rqlite/goose_store/dialect.go new file mode 100644 index 0000000..1735169 --- /dev/null +++ b/service/rqlite/goose_store/dialect.go @@ -0,0 +1,37 @@ +package goose_store + +import ( + "fmt" +) + +type Rqlite struct{} + +func (s *Rqlite) CreateTable(tableName string) string { + q := `CREATE TABLE %s ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + version_id INTEGER NOT NULL, + is_applied INTEGER NOT NULL, + tstamp DATETIME DEFAULT (datetime('now')) + )` + return fmt.Sprintf(q, tableName) +} + +func (s *Rqlite) InsertVersion(tableName string) string { + q := `INSERT INTO %s (version_id, is_applied) VALUES (?, ?)` + return fmt.Sprintf(q, tableName) +} + +func (s *Rqlite) DeleteVersion(tableName string) string { + q := `DELETE FROM %s WHERE version_id=?` + return fmt.Sprintf(q, tableName) +} + +func (s *Rqlite) GetMigrationByVersion(tableName string) string { + q := `SELECT tstamp, is_applied FROM %s WHERE version_id=? ORDER BY tstamp DESC LIMIT 1` + return fmt.Sprintf(q, tableName) +} + +func (s *Rqlite) ListMigrations(tableName string) string { + q := `SELECT version_id, is_applied from %s ORDER BY id DESC` + return fmt.Sprintf(q, tableName) +} diff --git a/service/rqlite/goose_store/store.go b/service/rqlite/goose_store/store.go new file mode 100644 index 0000000..e78eae4 --- /dev/null +++ b/service/rqlite/goose_store/store.go @@ -0,0 +1,105 @@ +package goose_store + +import ( + "context" + "database/sql" + "fmt" + + "github.com/pkg/errors" + "github.com/pressly/goose/v3/database" +) + +type Store struct { + tablename string + querier *Rqlite + db *sql.DB +} + +func NewStore(db *sql.DB) Store { + return Store{ + db: db, + tablename: "isp_config_service__goose_db_version", + querier: &Rqlite{}, + } +} + +func (s Store) Tablename() string { + return s.tablename +} + +func (s Store) CreateVersionTable(ctx context.Context, db database.DBTxConn) error { + q := s.querier.CreateTable(s.tablename) + if _, err := s.db.ExecContext(ctx, q); err != nil { + return fmt.Errorf("failed to create version table %q: %w", s.tablename, err) + } + return nil +} + +func (s Store) Insert(ctx context.Context, db database.DBTxConn, req database.InsertRequest) error { + q := s.querier.InsertVersion(s.tablename) + if _, err := s.db.ExecContext(ctx, q, req.Version, true); err != nil { + return fmt.Errorf("failed to insert version %d: %w", req.Version, err) + } + return nil +} + +func (s Store) Delete(ctx context.Context, db database.DBTxConn, version int64) error { + q := s.querier.DeleteVersion(s.tablename) + if _, err := s.db.ExecContext(ctx, q, version); err != nil { + return fmt.Errorf("failed to delete version %d: %w", version, err) + } + return nil +} + +func (s Store) GetMigration(ctx context.Context, db database.DBTxConn, version int64) (*database.GetMigrationResult, error) { + q := s.querier.GetMigrationByVersion(s.tablename) + var result database.GetMigrationResult + + rows, err := s.db.QueryContext(ctx, q, version) + if err != nil { + return nil, fmt.Errorf("failed to get migration %d: %w", version, err) + } + for rows.Next() { + isApplied := float64(0) + err := rows.Scan(&result.Timestamp, &isApplied) + if err != nil { + return nil, fmt.Errorf("failed to get migration %d: %w", version, err) + } + result.IsApplied = isApplied == 0 + return &result, nil + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to get migration %d: %w", version, err) + } + return nil, fmt.Errorf("%w: %d", database.ErrVersionNotFound, version) +} + +func (s Store) GetLatestVersion(ctx context.Context, db database.DBTxConn) (int64, error) { + return -1, errors.New("not implemented") +} + +func (s Store) ListMigrations(ctx context.Context, db database.DBTxConn) ([]*database.ListMigrationsResult, error) { + q := s.querier.ListMigrations(s.tablename) + rows, err := s.db.QueryContext(ctx, q) + if err != nil { + return nil, fmt.Errorf("failed to list migrations: %w", err) + } + defer rows.Close() + + var migrations []*database.ListMigrationsResult + for rows.Next() { + var result database.ListMigrationsResult + isApplied := float64(0) + version := float64(0) + if err := rows.Scan(&version, &isApplied); err != nil { + return nil, fmt.Errorf("failed to scan list migrations result: %w", err) + } + result.IsApplied = isApplied == 0 + result.Version = int64(version) + migrations = append(migrations, &result) + } + if err := rows.Err(); err != nil { + return nil, err + } + return migrations, nil +} diff --git a/service/rqlite/main.go b/service/rqlite/main.go new file mode 100644 index 0000000..03a9a33 --- /dev/null +++ b/service/rqlite/main.go @@ -0,0 +1,374 @@ +// Command rqlited is the rqlite server. +// nolint +package rqlite + +import ( + "context" + "crypto/tls" + "fmt" + "log" + "net" + "os" + "runtime" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/rqlite/rqlite/v8/auth" + "github.com/rqlite/rqlite/v8/cluster" + "github.com/rqlite/rqlite/v8/cmd" + "github.com/rqlite/rqlite/v8/db" + httpd "github.com/rqlite/rqlite/v8/http" + "github.com/rqlite/rqlite/v8/rtls" + "github.com/rqlite/rqlite/v8/store" + "github.com/rqlite/rqlite/v8/tcp" +) + +const name = `rqlite` +const desc = `rqlite is a lightweight, distributed relational database, which uses SQLite as its +storage engine. It provides an easy-to-use, fault-tolerant store for relational data. + +Visit https://www.rqlite.io to learn more.` + +func init() { + log.SetFlags(log.LstdFlags) + log.SetOutput(os.Stderr) + log.SetPrefix(fmt.Sprintf("[%s] ", name)) +} + +type localConfig struct { + Rqlite *Config +} + +func main(ctx context.Context, r *Rqlite) error { + mainCtx := ctx + + cfg, err := ParseFlags(name, desc, &BuildInfo{ + Version: cmd.Version, + Commit: cmd.Commit, + Branch: cmd.Branch, + SQLiteVersion: db.DBVersion, + }) + if err != nil { + log.Fatalf("failed to parse command-line flags: %s", err.Error()) + } + localCfg := localConfig{Rqlite: cfg} + err = r.cfg.Read(&localCfg) + if err != nil { + return errors.WithMessage(err, "read local config") + } + err = cfg.Validate() + if err != nil { + return errors.WithMessage(err, "validate rqlite configuration") + } + + r.localHttpAddr = cfg.HTTPAddr + + // Configure logging and pump out initial message. + log.Printf("%s starting, version %s, SQLite %s, commit %s, branch %s, compiler %s", name, cmd.Version, + db.DBVersion, cmd.Commit, cmd.Branch, runtime.Compiler) + log.Printf("%s, target architecture is %s, operating system target is %s", runtime.Version(), + runtime.GOARCH, runtime.GOOS) + log.Printf("launch command: %s", strings.Join(os.Args, " ")) + + // Create internode network mux and configure. + muxLn, err := net.Listen("tcp", cfg.RaftAddr) + if err != nil { + log.Fatalf("failed to listen on %s: %s", cfg.RaftAddr, err.Error()) + } + mux, err := startNodeMux(cfg, muxLn) + if err != nil { + log.Fatalf("failed to start node mux: %s", err.Error()) + } + + // Raft internode layer + raftLn := mux.Listen(cluster.MuxRaftHeader) + log.Printf("Raft TCP mux Listener registered with byte header %d", cluster.MuxRaftHeader) + raftDialer, err := cluster.CreateRaftDialer(cfg.NodeX509Cert, cfg.NodeX509Key, cfg.NodeX509CACert, + cfg.NodeVerifyServerName, cfg.NoNodeVerify) + if err != nil { + log.Fatalf("failed to create Raft dialer: %s", err.Error()) + } + raftTn := tcp.NewLayer(raftLn, raftDialer) + + // Create the store. + str, err := createStore(cfg, raftTn) + if err != nil { + log.Fatalf("failed to create store: %s", err.Error()) + } + + r.store = str + + // Get any credential store. + credStr, err := credentialStore(cfg) + if err != nil { + log.Fatalf("failed to get credential store: %s", err.Error()) + } + + // Create cluster service now, so nodes will be able to learn information about each other. + clstrServ, err := clusterService(cfg, mux.Listen(cluster.MuxClusterHeader), str, str, credStr) + if err != nil { + log.Fatalf("failed to create cluster service: %s", err.Error()) + } + log.Printf("cluster TCP mux Listener registered with byte header %d", cluster.MuxClusterHeader) + + // Create the HTTP service. + // + // We want to start the HTTP server as soon as possible, so the node is responsive and external + // systems can see that it's running. We still have to open the Store though, so the node won't + // be able to do much until that happens however. + clstrClient, err := createClusterClient(cfg, clstrServ) + if err != nil { + log.Fatalf("failed to create cluster client: %s", err.Error()) + } + httpServ, err := startHTTPService(cfg, str, clstrClient, credStr) + if err != nil { + log.Fatalf("failed to start HTTP server: %s", err.Error()) + } + + // Now, open store. How long this takes does depend on how much data is being stored by rqlite. + if err := str.Open(); err != nil { + log.Fatalf("failed to open store: %s", err.Error()) + } + + // Register remaining status providers. + httpServ.RegisterStatus("cluster", clstrServ) + httpServ.RegisterStatus("network", tcp.NetworkReporter{}) + + // Create the cluster! + nodes, err := str.Nodes() + if err != nil { + log.Fatalf("failed to get nodes %s", err.Error()) + } + if err := createCluster(mainCtx, cfg, len(nodes) > 0, clstrClient, str, httpServ, credStr); err != nil { + log.Fatalf("clustering failure: %s", err.Error()) + } + + // Tell the user the node is ready for HTTP, giving some advice on how to connect. + log.Printf("node HTTP API available at %s", cfg.HTTPURL()) + h, p, _ := net.SplitHostPort(cfg.HTTPAdv) + log.Printf("connect using the command-line tool via 'rqlite -H %s -p %s'", h, p) + + // Block until done. + <-mainCtx.Done() + + // Stop the HTTP server first, so clients get notification as soon as + // possible that the node is going away. + httpServ.Close() + + if cfg.RaftClusterRemoveOnShutdown { + remover := cluster.NewRemover(clstrClient, 5*time.Second, str) + remover.SetCredentials(cluster.CredentialsFor(credStr, cfg.JoinAs)) + log.Printf("initiating removal of this node from cluster before shutdown") + if err := remover.Do(cfg.NodeID, true); err != nil { + log.Fatalf("failed to remove this node from cluster before shutdown: %s", err.Error()) + } + log.Printf("removed this node successfully from cluster before shutdown") + } + + if cfg.RaftStepdownOnShutdown { + if str.IsLeader() { + // Don't log a confusing message if (probably) not Leader + log.Printf("stepping down as Leader before shutdown") + } + // Perform a stepdown, ignore any errors. + str.Stepdown(true) + } + + if err := str.Close(true); err != nil { + log.Printf("failed to close store: %s", err.Error()) + } + clstrServ.Close() + muxLn.Close() + return nil +} + +func createStore(cfg *Config, ln *tcp.Layer) (*store.Store, error) { + dbConf := store.NewDBConfig() + dbConf.OnDiskPath = cfg.OnDiskPath + dbConf.FKConstraints = cfg.FKConstraints + + str := store.New(ln, &store.Config{ + DBConf: dbConf, + Dir: cfg.DataPath, + ID: cfg.NodeID, + }) + + // Set optional parameters on store. + str.RaftLogLevel = cfg.RaftLogLevel + str.ShutdownOnRemove = cfg.RaftShutdownOnRemove + str.SnapshotThreshold = cfg.RaftSnapThreshold + str.SnapshotThresholdWALSize = cfg.RaftSnapThresholdWALSize + str.SnapshotInterval = cfg.RaftSnapInterval + str.LeaderLeaseTimeout = cfg.RaftLeaderLeaseTimeout + str.HeartbeatTimeout = cfg.RaftHeartbeatTimeout + str.ElectionTimeout = cfg.RaftElectionTimeout + str.ApplyTimeout = cfg.RaftApplyTimeout + str.BootstrapExpect = cfg.BootstrapExpect + str.ReapTimeout = cfg.RaftReapNodeTimeout + str.ReapReadOnlyTimeout = cfg.RaftReapReadOnlyNodeTimeout + str.AutoVacInterval = cfg.AutoVacInterval + + if store.IsNewNode(cfg.DataPath) { + log.Printf("no preexisting node state detected in %s, node may be bootstrapping", cfg.DataPath) + } else { + log.Printf("preexisting node state detected in %s", cfg.DataPath) + } + + return str, nil +} + +func startHTTPService(cfg *Config, str *store.Store, cltr *cluster.Client, credStr *auth.CredentialsStore) (*httpd.Service, error) { + // Create HTTP server and load authentication information. + s := httpd.New(cfg.HTTPAddr, str, cltr, credStr) + + s.CACertFile = cfg.HTTPx509CACert + s.CertFile = cfg.HTTPx509Cert + s.KeyFile = cfg.HTTPx509Key + s.ClientVerify = cfg.HTTPVerifyClient + s.DefaultQueueCap = cfg.WriteQueueCap + s.DefaultQueueBatchSz = cfg.WriteQueueBatchSz + s.DefaultQueueTimeout = cfg.WriteQueueTimeout + s.DefaultQueueTx = cfg.WriteQueueTx + s.AllowOrigin = cfg.HTTPAllowOrigin + s.BuildInfo = map[string]interface{}{ + "commit": cmd.Commit, + "branch": cmd.Branch, + "version": cmd.Version, + "compiler": runtime.Compiler, + "build_time": cmd.Buildtime, + } + return s, s.Start() +} + +// startNodeMux starts the TCP mux on the given listener, which should be already +// bound to the relevant interface. +func startNodeMux(cfg *Config, ln net.Listener) (*tcp.Mux, error) { + var err error + adv := tcp.NameAddress{ + Address: cfg.RaftAdv, + } + + var mux *tcp.Mux + if cfg.NodeX509Cert != "" { + var b strings.Builder + b.WriteString(fmt.Sprintf("enabling node-to-node encryption with cert: %s, key: %s", + cfg.NodeX509Cert, cfg.NodeX509Key)) + if cfg.NodeX509CACert != "" { + b.WriteString(fmt.Sprintf(", CA cert %s", cfg.NodeX509CACert)) + } + if cfg.NodeVerifyClient { + b.WriteString(", mutual TLS enabled") + } else { + b.WriteString(", mutual TLS disabled") + } + log.Println(b.String()) + mux, err = tcp.NewTLSMux(ln, adv, cfg.NodeX509Cert, cfg.NodeX509Key, cfg.NodeX509CACert, + cfg.NoNodeVerify, cfg.NodeVerifyClient) + } else { + mux, err = tcp.NewMux(ln, adv) + } + if err != nil { + return nil, fmt.Errorf("failed to create node-to-node mux: %s", err.Error()) + } + go mux.Serve() + return mux, nil +} + +func credentialStore(cfg *Config) (*auth.CredentialsStore, error) { + if cfg.AuthFile == "" { + return nil, nil + } + return auth.NewCredentialsStoreFromFile(cfg.AuthFile) +} + +func clusterService(cfg *Config, ln net.Listener, db cluster.Database, mgr cluster.Manager, credStr *auth.CredentialsStore) (*cluster.Service, error) { + c := cluster.New(ln, db, mgr, credStr) + c.SetAPIAddr(cfg.HTTPAdv) + c.EnableHTTPS(cfg.HTTPx509Cert != "" && cfg.HTTPx509Key != "") // Conditions met for an HTTPS API + if err := c.Open(); err != nil { + return nil, err + } + return c, nil +} + +func createClusterClient(cfg *Config, clstr *cluster.Service) (*cluster.Client, error) { + var dialerTLSConfig *tls.Config + var err error + if cfg.NodeX509Cert != "" || cfg.NodeX509CACert != "" { + dialerTLSConfig, err = rtls.CreateClientConfig(cfg.NodeX509Cert, cfg.NodeX509Key, + cfg.NodeX509CACert, cfg.NodeVerifyServerName, cfg.NoNodeVerify) + if err != nil { + return nil, fmt.Errorf("failed to create TLS config for cluster dialer: %s", err.Error()) + } + } + clstrDialer := tcp.NewDialer(cluster.MuxClusterHeader, dialerTLSConfig) + clstrClient := cluster.NewClient(clstrDialer, cfg.ClusterConnectTimeout) + if err := clstrClient.SetLocal(cfg.RaftAdv, clstr); err != nil { + return nil, fmt.Errorf("failed to set cluster client local parameters: %s", err.Error()) + } + return clstrClient, nil +} + +func createCluster(ctx context.Context, cfg *Config, hasPeers bool, client *cluster.Client, str *store.Store, + httpServ *httpd.Service, credStr *auth.CredentialsStore) error { + joins := cfg.JoinAddresses() + if err := networkCheckJoinAddrs(joins); err != nil { + return err + } + if joins == nil && cfg.DiscoMode == "" && !hasPeers { + if cfg.RaftNonVoter { + return fmt.Errorf("cannot create a new non-voting node without joining it to an existing cluster") + } + + // Brand new node, told to bootstrap itself. So do it. + log.Println("bootstrapping single new node") + if err := str.Bootstrap(store.NewServer(str.ID(), cfg.RaftAdv, true)); err != nil { + return fmt.Errorf("failed to bootstrap single new node: %s", err.Error()) + } + return nil + } + + // Prepare definition of being part of a cluster. + bootDoneFn := func() bool { + leader, _ := str.LeaderAddr() + return leader != "" + } + clusterSuf := cluster.VoterSuffrage(!cfg.RaftNonVoter) + + joiner := cluster.NewJoiner(client, cfg.JoinAttempts, cfg.JoinInterval) + joiner.SetCredentials(cluster.CredentialsFor(credStr, cfg.JoinAs)) + if joins != nil && cfg.BootstrapExpect == 0 { + // Explicit join operation requested, so do it. + j, err := joiner.Do(ctx, joins, str.ID(), cfg.RaftAdv, clusterSuf) + if err != nil { + return fmt.Errorf("failed to join cluster: %s", err.Error()) + } + log.Println("successfully joined cluster at", j) + return nil + } + + if joins != nil && cfg.BootstrapExpect > 0 { + // Bootstrap with explicit join addresses requests. + bs := cluster.NewBootstrapper(cluster.NewAddressProviderString(joins), client) + bs.SetCredentials(cluster.CredentialsFor(credStr, cfg.JoinAs)) + return bs.Boot(ctx, str.ID(), cfg.RaftAdv, clusterSuf, bootDoneFn, cfg.BootstrapExpectTimeout) + } + + if cfg.DiscoMode == "" { + // No more clustering techniques to try. Node will just sit, probably using + // existing Raft state. + return nil + } + return nil +} + +func networkCheckJoinAddrs(joinAddrs []string) error { + if len(joinAddrs) > 0 { + log.Println("checking that supplied join addresses don't serve HTTP(S)") + if addr, ok := httpd.AnyServingHTTP(joinAddrs); ok { + return fmt.Errorf("join address %s appears to be serving HTTP when it should be Raft", addr) + } + } + return nil +} diff --git a/service/rqlite/runner.go b/service/rqlite/runner.go new file mode 100644 index 0000000..1c9286b --- /dev/null +++ b/service/rqlite/runner.go @@ -0,0 +1,76 @@ +package rqlite + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/pkg/errors" + _ "github.com/rqlite/gorqlite/stdlib" + "github.com/rqlite/rqlite/v8/store" + "github.com/txix-open/isp-kit/config" +) + +var ( + ErrNotRun = errors.New("rqlite runner is not initialized") +) + +type Rqlite struct { + cfg *config.Config + + localHttpAddr string + store *store.Store + closed chan struct{} + cancel context.CancelFunc +} + +func New(cfg *config.Config) *Rqlite { + return &Rqlite{ + cfg: cfg, + closed: make(chan struct{}), + } +} + +func (r *Rqlite) Run(ctx context.Context) error { + ctx, r.cancel = context.WithCancel(ctx) + defer close(r.closed) + + return main(ctx, r) +} + +func (r *Rqlite) WaitForLeader(timeout time.Duration) error { + if r.store == nil { + return ErrNotRun + } + _, err := r.store.WaitForLeader(timeout) + if err != nil { + return errors.WithMessage(err, "wait for leader in rqlite store") + } + return nil +} + +func (r *Rqlite) IsLeader() bool { + if r.store == nil { + return false + } + return r.store.IsLeader() +} + +func (r *Rqlite) SqlDB() (*sql.DB, error) { + if r.store == nil { + return nil, ErrNotRun + } + + return sql.Open("rqlite", r.Dsn()) +} + +func (r *Rqlite) Dsn() string { + return fmt.Sprintf("http://%s", r.localHttpAddr) +} + +func (r *Rqlite) Close() error { + r.cancel() + <-r.closed + return nil +} diff --git a/service/schema.go b/service/schema.go deleted file mode 100644 index 9c5c42f..0000000 --- a/service/schema.go +++ /dev/null @@ -1,36 +0,0 @@ -package service - -import ( - log "github.com/integration-system/isp-log" - "github.com/pkg/errors" - "isp-config-service/codes" - "isp-config-service/entity" - "isp-config-service/holder" - "isp-config-service/model" - "isp-config-service/store/state" -) - -var ( - Schema schemaService -) - -type schemaService struct{} - -func (schemaService) HandleUpdateConfigSchemaCommand(schema entity.ConfigSchema, state state.WritableState) error { - module := state.Modules().GetById(schema.ModuleId) - if module == nil { - return errors.Errorf("module with id %s not found", schema.ModuleId) - } - - schema = state.WritableSchemas().Upsert(schema) - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - _, err := model.SchemaRep.Upsert(schema) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "schema": schema, - }).Errorf(codes.DatabaseOperationError, "upsert schema: %v", err) - } - } - return nil -} diff --git a/service/startup/service.go b/service/startup/service.go new file mode 100644 index 0000000..a4ee2a3 --- /dev/null +++ b/service/startup/service.go @@ -0,0 +1,94 @@ +package startup + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/pressly/goose/v3" + "github.com/txix-open/isp-kit/bootstrap" + "github.com/txix-open/isp-kit/dbx/migration" + "github.com/txix-open/isp-kit/http/httpclix" + "github.com/txix-open/isp-kit/log" + "isp-config-service/assembly" + "isp-config-service/service/rqlite" + "isp-config-service/service/rqlite/db" + "isp-config-service/service/rqlite/goose_store" +) + +const ( + waitForLeaderTimeout = 30 * time.Second +) + +type Service struct { + boot *bootstrap.Bootstrap + logger log.Logger + rqlite *rqlite.Rqlite +} + +func New(boot *bootstrap.Bootstrap) Service { + rqlite := rqlite.New(boot.App.Config()) + return Service{ + boot: boot, + logger: boot.App.Logger(), + rqlite: rqlite, + } +} + +func (s Service) Run(ctx context.Context) error { + go func() { + err := s.rqlite.Run(ctx) + if err != nil { + s.boot.Fatal(errors.WithMessage(err, "run embedded rqlite")) + } + }() + time.Sleep(1 * time.Second) //optimistically wait for store initialization + + s.logger.Debug(ctx, fmt.Sprintf("waiting for cluster startup for %s...", waitForLeaderTimeout)) + err := s.rqlite.WaitForLeader(waitForLeaderTimeout) + if err != nil { + return errors.WithMessage(err, "wait for leader") + } + + if s.rqlite.IsLeader() { + s.logger.Debug(ctx, "is a leader") + err := s.leaderStartup(ctx) + if err != nil { + return errors.WithMessage(err, "start leader") + } + } else { + s.logger.Debug(ctx, "is not a leader") + } + + db, err := db.Open(ctx, s.rqlite.Dsn(), httpclix.Default()) + if err != nil { + return errors.WithMessage(err, "dial to embedded rqlite") + } + + _ = assembly.NewLocator(s.logger, db) + + return nil +} + +func (s Service) Close() error { + _ = s.rqlite.Close() + return nil +} + +func (s Service) leaderStartup(ctx context.Context) error { + db, err := s.rqlite.SqlDB() + if err != nil { + return errors.WithMessage(err, "open sql db") + } + defer db.Close() + + migrationRunner := migration.NewRunner("", s.boot.MigrationsDir, s.logger) + rqliteGooseStore := goose_store.NewStore(db) + err = migrationRunner.Run(ctx, db, goose.WithStore(rqliteGooseStore)) + if err != nil { + return errors.WithMessage(err, "apply migrations") + } + + return nil +} diff --git a/service/version.go b/service/version.go deleted file mode 100644 index 7cc9adf..0000000 --- a/service/version.go +++ /dev/null @@ -1,75 +0,0 @@ -package service - -import ( - "time" - - log "github.com/integration-system/isp-log" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/domain" - "isp-config-service/entity" - "isp-config-service/holder" - "isp-config-service/model" - "isp-config-service/store/state" -) - -var ( - ConfigHistory = configHistoryService{} -) - -type configHistoryService struct{} - -func (configHistoryService) HandleDeleteVersionConfigCommand(cfg cluster.Identity, - state state.WritableState) cluster.ResponseWithError { - state.WriteableVersionConfigStore().Delete(cfg.Id) - var ( - deleted int - err error - ) - if holder.ClusterClient.IsLeader() { - // TODO handle db errors - deleted, err = model.VersionStoreRep.Delete(cfg.Id) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "versionConfigId": cfg.Id, - }).Errorf(codes.DatabaseOperationError, "delete configs: %v", err) - } - } - return cluster.NewResponse(domain.DeleteResponse{Deleted: deleted}) -} - -func (configHistoryService) GetAllVersionConfigById(id string, state state.ReadonlyState) []entity.VersionConfig { - return state.VersionConfig().GetByConfigId(id) -} - -func (s configHistoryService) SaveConfigVersion(id string, oldConfig entity.Config, - state state.WritableState, createdAt time.Time) { - cfg := entity.VersionConfig{ - Id: id, - ConfigVersion: oldConfig.Version, - ConfigId: oldConfig.Id, - Data: oldConfig.Data, - CreatedAt: createdAt, - } - removedVersionId := state.WriteableVersionConfigStore().Update(cfg) - if holder.ClusterClient.IsLeader() { - s.updateDB(cfg, removedVersionId) - } -} - -func (configHistoryService) updateDB(cfg entity.VersionConfig, removedId string) { - _, err := model.VersionStoreRep.Upsert(cfg) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "version_config": cfg, - }).Errorf(codes.DatabaseOperationError, "upsert version config: %v", err) - } - if removedId != "" { - _, err := model.VersionStoreRep.Delete(removedId) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "version_config_id": removedId, - }).Errorf(codes.DatabaseOperationError, "delete version config: %v", err) - } - } -} diff --git a/store/apply.go b/store/apply.go deleted file mode 100644 index 3c52455..0000000 --- a/store/apply.go +++ /dev/null @@ -1,147 +0,0 @@ -package store - -import ( - "time" - - "github.com/integration-system/isp-lib/v2/structure" - "github.com/pkg/errors" - "isp-config-service/cluster" - "isp-config-service/domain" - "isp-config-service/entity" - "isp-config-service/service" -) - -func (s *Store) applyUpdateBackendDeclarationCommand(data []byte) (interface{}, error) { - declaration := structure.BackendDeclaration{} - err := json.Unmarshal(data, &declaration) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal structure.BackendDeclaration") - } - service.ClusterMesh.HandleUpdateBackendDeclarationCommand(declaration, s.state) - return nil, nil -} - -func (s *Store) applyDeleteBackendDeclarationCommand(data []byte) (interface{}, error) { - declaration := structure.BackendDeclaration{} - err := json.Unmarshal(data, &declaration) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal structure.BackendDeclaration") - } - service.ClusterMesh.HandleDeleteBackendDeclarationCommand(declaration, s.state) - return nil, nil -} - -func (s *Store) applyUpdateConfigSchemaCommand(data []byte) (interface{}, error) { - schema := entity.ConfigSchema{} - err := json.Unmarshal(data, &schema) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal entity.ConfigSchema") - } - err = service.Schema.HandleUpdateConfigSchemaCommand(schema, s.state) - if err != nil { - return nil, errors.WithMessage(err, "update config schema") - } - return nil, nil -} - -func (s *Store) applyModuleConnectedCommand(data []byte) (interface{}, error) { - module := entity.Module{} - err := json.Unmarshal(data, &module) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal entity.Module") - } - service.ModuleRegistry.HandleModuleConnectedCommand(module, s.state) - return nil, nil -} - -func (s *Store) applyModuleDisconnectedCommand(data []byte) (interface{}, error) { - module := entity.Module{} - err := json.Unmarshal(data, &module) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal entity.Module") - } - service.ModuleRegistry.HandleModuleDisconnectedCommand(module, s.state) - return nil, nil -} - -func (s *Store) applyDeleteModulesCommand(data []byte) (interface{}, error) { - deleteModules := cluster.DeleteModules{} - err := json.Unmarshal(data, &deleteModules) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal cluster.DeleteModules") - } - deleted := service.ModuleRegistry.HandleDeleteModulesCommand(deleteModules, s.state) - return domain.DeleteResponse{Deleted: deleted}, nil -} - -func (s *Store) applyActivateConfigCommand(data []byte) (interface{}, error) { - activateConfig := cluster.ActivateConfig{} - err := json.Unmarshal(data, &activateConfig) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal cluster.ActivateConfig") - } - response := service.ConfigService.HandleActivateConfigCommand(activateConfig, s.state) - return response, nil -} - -func (s *Store) applyDeleteConfigsCommand(data []byte) (interface{}, error) { - deleteConfigs := cluster.DeleteConfigs{} - err := json.Unmarshal(data, &deleteConfigs) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal cluster.DeleteConfigs") - } - deleted := service.ConfigService.HandleDeleteConfigsCommand(deleteConfigs, s.state) - return domain.DeleteResponse{Deleted: deleted}, nil -} - -func (s *Store) applyUpsertConfigCommand(data []byte) (interface{}, error) { - config := cluster.UpsertConfig{} - err := json.Unmarshal(data, &config) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal cluster.UpsertConfig") - } - response := service.ConfigService.HandleUpsertConfigCommand(config, s.state) - return response, nil -} - -func (s *Store) applyDeleteCommonConfigsCommand(data []byte) (interface{}, error) { - deleteConfigs := cluster.DeleteCommonConfig{} - err := json.Unmarshal(data, &deleteConfigs) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal cluster.DeleteCommonConfig") - } - response := service.CommonConfig.HandleDeleteConfigsCommand(deleteConfigs, s.state) - return response, nil -} - -func (s *Store) applyUpsertCommonConfigCommand(data []byte) (interface{}, error) { - config := cluster.UpsertCommonConfig{} - err := json.Unmarshal(data, &config) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal cluster.UpsertCommonConfig") - } - response := service.CommonConfig.HandleUpsertConfigCommand(config, s.state) - return response, nil -} - -func (s *Store) applyBroadcastEventCommand(data []byte) (interface{}, error) { - event := cluster.BroadcastEvent{} - err := json.Unmarshal(data, &event) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal cluster.BroadcastEvent") - } - if event.PerformUntil.After(time.Now().UTC()) { - service.Discovery.BroadcastEvent(event) - } - return nil, nil -} - -func (s *Store) applyDeleteVersionConfigCommand(data []byte) (interface{}, error) { - cfg := cluster.Identity{} - err := json.Unmarshal(data, &cfg) - if err != nil { - return nil, errors.WithMessage(err, "unmarshal cluster.Identity") - } - response := service.ConfigHistory.HandleDeleteVersionConfigCommand(cfg, s.state) - return response, nil -} diff --git a/store/restore.go b/store/restore.go deleted file mode 100644 index b628efa..0000000 --- a/store/restore.go +++ /dev/null @@ -1,34 +0,0 @@ -package store - -import ( - "github.com/pkg/errors" - "isp-config-service/model" - "isp-config-service/store/state" -) - -func NewStateFromRepository() (*state.State, error) { - configs, err := model.ConfigRep.Snapshot() - if err != nil { - return nil, errors.WithMessage(err, "restore configs") - } - schemas, err := model.SchemaRep.Snapshot() - if err != nil { - return nil, errors.WithMessage(err, "load config schemas") - } - modules, err := model.ModuleRep.Snapshot() - if err != nil { - return nil, errors.WithMessage(err, "load modules registry") - } - commonConfigs, err := model.CommonConfigRep.Snapshot() - if err != nil { - return nil, errors.WithMessage(err, "load common configs") - } - - versionStore, err := model.VersionStoreRep.Snapshot() - if err != nil { - return nil, errors.WithMessage(err, "load version store") - } - - st := state.NewStateFromSnapshot(configs, schemas, modules, commonConfigs, versionStore) - return st, nil -} diff --git a/store/state/common_config.go b/store/state/common_config.go deleted file mode 100644 index 45ec482..0000000 --- a/store/state/common_config.go +++ /dev/null @@ -1,88 +0,0 @@ -package state - -import ( - "isp-config-service/entity" -) - -type WriteableCommonConfigStore interface { - ReadonlyCommonConfigStore - Create(config entity.CommonConfig) entity.CommonConfig - UpdateById(config entity.CommonConfig) - DeleteByIds(ids []string) int -} - -type ReadonlyCommonConfigStore interface { - GetByIds(ids []string) []entity.CommonConfig - GetByName(name string) []entity.CommonConfig - GetAll() []entity.CommonConfig -} - -type CommonConfigStore struct { - Configs []entity.CommonConfig -} - -func (cs CommonConfigStore) GetByIds(ids []string) []entity.CommonConfig { - idsMap := StringsToMap(ids) - configs := make([]entity.CommonConfig, 0, len(ids)) - for _, config := range cs.Configs { - if _, found := idsMap[config.Id]; found { - configs = append(configs, config) - } - } - return configs -} - -func (cs CommonConfigStore) GetByName(name string) []entity.CommonConfig { - configs := make([]entity.CommonConfig, 0) - for _, config := range cs.Configs { - if config.Name == name { - configs = append(configs, config) - } - } - return configs -} - -func (cs *CommonConfigStore) Create(config entity.CommonConfig) entity.CommonConfig { - if config.Name == "" { - config.Name = configDefaultName - } - if config.Data == nil { - config.Data = make(entity.ConfigData) - } - cs.Configs = append(cs.Configs, config) - return config -} - -func (cs *CommonConfigStore) UpdateById(config entity.CommonConfig) { - for i := range cs.Configs { - if cs.Configs[i].Id == config.Id { - cs.Configs[i] = config - break - } - } -} - -func (cs *CommonConfigStore) DeleteByIds(ids []string) int { - idsMap := StringsToMap(ids) - var deleted int - for i := 0; i < len(cs.Configs); i++ { - id := cs.Configs[i].Id - if _, ok := idsMap[id]; ok { - // change configs ordering - cs.Configs[i] = cs.Configs[len(cs.Configs)-1] - cs.Configs = cs.Configs[:len(cs.Configs)-1] - deleted++ - } - } - return deleted -} - -func (cs *CommonConfigStore) GetAll() []entity.CommonConfig { - response := make([]entity.CommonConfig, len(cs.Configs)) - copy(response, cs.Configs) - return response -} - -func NewCommonConfigStore() *CommonConfigStore { - return &CommonConfigStore{Configs: make([]entity.CommonConfig, 0)} -} diff --git a/store/state/configs.go b/store/state/configs.go deleted file mode 100644 index 91fe0ec..0000000 --- a/store/state/configs.go +++ /dev/null @@ -1,155 +0,0 @@ -package state - -import ( - "time" - - "isp-config-service/entity" -) - -const ( - configDefaultName = "unnamed" -) - -type WriteableConfigStore interface { - ReadonlyConfigStore - Create(config entity.Config) entity.Config - UpdateById(config entity.Config) - DeleteByIds(ids []string) int - Activate(config entity.Config, date time.Time) (affected []entity.Config) -} - -type ReadonlyConfigStore interface { - GetActiveByModuleId(moduleId string) *entity.Config - GetByIds(ids []string) []entity.Config - GetByModuleIds(ids []string) []entity.Config - FilterByCommonConfigs(commonIds []string) []entity.Config -} - -type ConfigStore struct { - Configs []entity.Config -} - -func (cs ConfigStore) GetActiveByModuleId(moduleId string) *entity.Config { - for i := range cs.Configs { - if cs.Configs[i].ModuleId == moduleId && cs.Configs[i].Active { - conf := cs.Configs[i] - return &conf - } - } - return nil -} - -func (cs ConfigStore) GetByIds(ids []string) []entity.Config { - idsMap := StringsToMap(ids) - result := make([]entity.Config, 0, len(ids)) - for i := range cs.Configs { - if _, ok := idsMap[cs.Configs[i].Id]; ok { - result = append(result, cs.Configs[i]) - } - } - return result -} - -func (cs ConfigStore) GetByModuleIds(ids []string) []entity.Config { - idsMap := StringsToMap(ids) - result := make([]entity.Config, 0, len(ids)) - for i := range cs.Configs { - if _, ok := idsMap[cs.Configs[i].ModuleId]; ok { - result = append(result, cs.Configs[i]) - } - } - return result -} - -func (cs *ConfigStore) Create(config entity.Config) entity.Config { - config.Version = cs.calcNewVersion(config.ModuleId) - if config.Version == 1 { - config.Active = true - } - if config.Name == "" { - config.Name = configDefaultName - } - if config.Data == nil { - config.Data = make(entity.ConfigData) - } - if config.CommonConfigs == nil { - config.CommonConfigs = make([]string, 0) - } - cs.Configs = append(cs.Configs, config) - return config -} - -func (cs *ConfigStore) UpdateById(config entity.Config) { - for i := range cs.Configs { - if cs.Configs[i].Id == config.Id { - cs.Configs[i] = config - break - } - } -} - -func (cs *ConfigStore) calcNewVersion(moduleId string) int32 { - var maxVersion int32 = 0 - for i := range cs.Configs { - if cs.Configs[i].ModuleId == moduleId { - if cs.Configs[i].Version > maxVersion { - maxVersion = cs.Configs[i].Version - } - } - } - return maxVersion + 1 -} - -func (cs *ConfigStore) DeleteByIds(ids []string) int { - idsMap := StringsToMap(ids) - var deleted int - for i := 0; i < len(cs.Configs); i++ { - id := cs.Configs[i].Id - if _, ok := idsMap[id]; ok { - // change configs ordering - cs.Configs[i] = cs.Configs[len(cs.Configs)-1] - cs.Configs = cs.Configs[:len(cs.Configs)-1] - deleted++ - } - } - return deleted -} - -func (cs *ConfigStore) Activate(config entity.Config, date time.Time) []entity.Config { - affected := cs.deactivate(config.ModuleId, date) - config.Active = true - config.UpdatedAt = date - cs.UpdateById(config) - affected = append(affected, config) - return affected -} - -func (cs *ConfigStore) deactivate(moduleId string, date time.Time) []entity.Config { - affected := make([]entity.Config, 0) - for i := range cs.Configs { - if cs.Configs[i].ModuleId == moduleId && cs.Configs[i].Active { - cs.Configs[i].Active = false - cs.Configs[i].UpdatedAt = date - affected = append(affected, cs.Configs[i]) - } - } - return affected -} - -func (cs *ConfigStore) FilterByCommonConfigs(commonIds []string) []entity.Config { - idsMap := StringsToMap(commonIds) - result := make([]entity.Config, 0) - for i := range cs.Configs { - for _, commonId := range cs.Configs[i].CommonConfigs { - if _, ok := idsMap[commonId]; ok { - result = append(result, cs.Configs[i]) - break - } - } - } - return result -} - -func NewConfigStore() *ConfigStore { - return &ConfigStore{Configs: make([]entity.Config, 0)} -} diff --git a/store/state/mesh.go b/store/state/mesh.go deleted file mode 100644 index ba911cb..0000000 --- a/store/state/mesh.go +++ /dev/null @@ -1,95 +0,0 @@ -package state - -import ( - "github.com/integration-system/isp-lib/v2/structure" -) - -type WriteableMesh interface { - ReadonlyMesh - UpsertBackend(backend structure.BackendDeclaration) (changed bool) - DeleteBackend(backend structure.BackendDeclaration) (deleted bool) -} - -type ReadonlyMesh interface { - GetModuleAddresses(moduleName string) []structure.AddressConfiguration - GetBackends(module string) []structure.BackendDeclaration - GetRoutes() structure.RoutingConfig -} - -type NodesMap map[string]structure.BackendDeclaration - -type Mesh struct { - // { "ModuleName": { "address": BackendDeclaration, ...}, ... } - ModulesMap map[string]NodesMap -} - -func NewMesh() *Mesh { - return &Mesh{ModulesMap: make(map[string]NodesMap)} -} - -func (m Mesh) GetBackends(module string) []structure.BackendDeclaration { - declarations := make([]structure.BackendDeclaration, 0) - if nodes, ok := m.ModulesMap[module]; ok { - for _, backend := range nodes { - declarations = append(declarations, backend) - } - } - return declarations -} - -func (m Mesh) GetModuleAddresses(moduleName string) []structure.AddressConfiguration { - addressList := make([]structure.AddressConfiguration, 0) - if nodes, ok := m.ModulesMap[moduleName]; ok { - for _, backend := range nodes { - addressList = append(addressList, backend.Address) - } - } - return addressList -} - -func (m Mesh) GetRoutes() structure.RoutingConfig { - routes := structure.RoutingConfig{} - for _, nodes := range m.ModulesMap { - for _, backend := range nodes { - routes = append(routes, backend) - } - } - return routes -} - -func (m *Mesh) UpsertBackend(backend structure.BackendDeclaration) bool { - address := backend.Address.GetAddress() - - nodes, ok := m.ModulesMap[backend.ModuleName] - if !ok { - m.ModulesMap[backend.ModuleName] = NodesMap{address: backend} - return true - } - - old, ok := nodes[address] - if !ok { - nodes[address] = backend - return true - } - - if old.Version != backend.Version || !old.IsPathsEqual(backend.Endpoints) || old.LibVersion != backend.LibVersion { - nodes[address] = backend - return true - } - - return false -} - -func (m *Mesh) DeleteBackend(backend structure.BackendDeclaration) (deleted bool) { - address := backend.Address.GetAddress() - if nodes, ok := m.ModulesMap[backend.ModuleName]; ok { - if _, ok := nodes[address]; ok { - delete(nodes, address) - deleted = true - } - if len(nodes) == 0 { - delete(m.ModulesMap, backend.ModuleName) - } - } - return -} diff --git a/store/state/modules.go b/store/state/modules.go deleted file mode 100644 index d11c2e5..0000000 --- a/store/state/modules.go +++ /dev/null @@ -1,78 +0,0 @@ -package state - -import ( - "isp-config-service/entity" -) - -type WriteableModuleStore interface { - ReadonlyModuleStore - UpdateByName(module entity.Module) - Create(entity.Module) - DeleteByIds(ids []string) (deleted []entity.Module) -} - -type ReadonlyModuleStore interface { - GetByName(name string) *entity.Module - GetById(id string) *entity.Module - GetAll() []entity.Module -} - -type ModuleStore struct { - Modules []entity.Module -} - -func (ms ModuleStore) GetByName(name string) *entity.Module { - for _, module := range ms.Modules { - if module.Name == name { - return &module - } - } - return nil -} - -func (ms ModuleStore) GetById(id string) *entity.Module { - for _, module := range ms.Modules { - if module.Id == id { - return &module - } - } - return nil -} - -func (ms *ModuleStore) UpdateByName(module entity.Module) { - for i := range ms.Modules { - if ms.Modules[i].Name == module.Name { - ms.Modules[i] = module - return - } - } -} - -func (ms *ModuleStore) Create(module entity.Module) { - ms.Modules = append(ms.Modules, module) -} - -func (ms *ModuleStore) DeleteByIds(ids []string) []entity.Module { - idsMap := StringsToMap(ids) - var deleted []entity.Module - for i := 0; i < len(ms.Modules); i++ { - id := ms.Modules[i].Id - if _, ok := idsMap[id]; ok { - // change modules ordering - deleted = append(deleted, ms.Modules[i]) - ms.Modules[i] = ms.Modules[len(ms.Modules)-1] - ms.Modules = ms.Modules[:len(ms.Modules)-1] - } - } - return deleted -} - -func (ms *ModuleStore) GetAll() []entity.Module { - response := make([]entity.Module, len(ms.Modules)) - copy(response, ms.Modules) - return response -} - -func NewModuleStore() *ModuleStore { - return &ModuleStore{Modules: make([]entity.Module, 0)} -} diff --git a/store/state/schemas.go b/store/state/schemas.go deleted file mode 100644 index 9b24018..0000000 --- a/store/state/schemas.go +++ /dev/null @@ -1,62 +0,0 @@ -package state - -import ( - "isp-config-service/entity" -) - -type WriteableSchemaStore interface { - ReadonlySchemaStore - Upsert(schema entity.ConfigSchema) entity.ConfigSchema - DeleteByIds(ids []string) int -} - -type ReadonlySchemaStore interface { - GetByModuleIds(ids []string) []entity.ConfigSchema -} - -type SchemaStore struct { - Schemas []entity.ConfigSchema -} - -func (ss SchemaStore) GetByModuleIds(ids []string) []entity.ConfigSchema { - idsMap := StringsToMap(ids) - result := make([]entity.ConfigSchema, 0, len(ids)) - for _, schema := range ss.Schemas { - if _, ok := idsMap[schema.ModuleId]; ok { - result = append(result, schema) - } - } - return result -} - -func (ss *SchemaStore) DeleteByIds(ids []string) int { - idsMap := StringsToMap(ids) - var deleted int - for i := 0; i < len(ss.Schemas); i++ { - id := ss.Schemas[i].Id - if _, ok := idsMap[id]; ok { - // change schemas ordering - ss.Schemas[i] = ss.Schemas[len(ss.Schemas)-1] - ss.Schemas = ss.Schemas[:len(ss.Schemas)-1] - deleted++ - } - } - return deleted -} - -func (ss *SchemaStore) Upsert(schema entity.ConfigSchema) entity.ConfigSchema { - for key, value := range ss.Schemas { - if value.ModuleId == schema.ModuleId { - schema.Id = value.Id - schema.CreatedAt = value.CreatedAt - ss.Schemas[key] = schema - return schema - } - } - ss.Schemas = append(ss.Schemas, schema) - return schema -} - -func NewSchemaStore() *SchemaStore { - return &SchemaStore{Schemas: make([]entity.ConfigSchema, 0)} -} diff --git a/store/state/state.go b/store/state/state.go deleted file mode 100644 index f914942..0000000 --- a/store/state/state.go +++ /dev/null @@ -1,104 +0,0 @@ -package state - -import ( - "isp-config-service/entity" -) - -type State struct { - MeshStore *Mesh - ConfigsStore *ConfigStore - SchemasStore *SchemaStore - ModulesStore *ModuleStore - CommonConfigsStore *CommonConfigStore - VersionConfigStore *VersionConfigStore -} - -func (s State) Mesh() ReadonlyMesh { - return s.MeshStore -} - -func (s State) Configs() ReadonlyConfigStore { - return s.ConfigsStore -} - -func (s State) Schemas() ReadonlySchemaStore { - return s.SchemasStore -} - -func (s State) Modules() ReadonlyModuleStore { - return s.ModulesStore -} - -func (s State) CommonConfigs() ReadonlyCommonConfigStore { - return s.CommonConfigsStore -} - -func (s State) VersionConfig() ReadonlyVersionConfigStore { - return s.VersionConfigStore -} - -func (s *State) WritableMesh() WriteableMesh { - return s.MeshStore -} - -func (s *State) WritableConfigs() WriteableConfigStore { - return s.ConfigsStore -} - -func (s *State) WritableSchemas() WriteableSchemaStore { - return s.SchemasStore -} - -func (s *State) WritableModules() WriteableModuleStore { - return s.ModulesStore -} - -func (s *State) WritableCommonConfigs() WriteableCommonConfigStore { - return s.CommonConfigsStore -} - -func (s *State) WriteableVersionConfigStore() WriteableVersionConfigStore { - return s.VersionConfigStore -} - -type WritableState interface { - ReadonlyState - WritableMesh() WriteableMesh - WritableConfigs() WriteableConfigStore - WritableSchemas() WriteableSchemaStore - WritableModules() WriteableModuleStore - WritableCommonConfigs() WriteableCommonConfigStore - WriteableVersionConfigStore() WriteableVersionConfigStore -} - -type ReadonlyState interface { - Mesh() ReadonlyMesh - Configs() ReadonlyConfigStore - Schemas() ReadonlySchemaStore - Modules() ReadonlyModuleStore - CommonConfigs() ReadonlyCommonConfigStore - VersionConfig() ReadonlyVersionConfigStore -} - -func NewState() *State { - return &State{ - MeshStore: NewMesh(), - ConfigsStore: NewConfigStore(), - SchemasStore: NewSchemaStore(), - ModulesStore: NewModuleStore(), - CommonConfigsStore: NewCommonConfigStore(), - VersionConfigStore: NewVersionConfigStore(), - } -} - -func NewStateFromSnapshot(configs []entity.Config, schemas []entity.ConfigSchema, - modules []entity.Module, commConfigs []entity.CommonConfig, versionStore []entity.VersionConfig) *State { - return &State{ - MeshStore: NewMesh(), - ConfigsStore: &ConfigStore{Configs: configs}, - SchemasStore: &SchemaStore{Schemas: schemas}, - ModulesStore: &ModuleStore{Modules: modules}, - CommonConfigsStore: &CommonConfigStore{Configs: commConfigs}, - VersionConfigStore: NewVersionConfigStoreFromSnapshot(versionStore), - } -} diff --git a/store/state/utils.go b/store/state/utils.go deleted file mode 100644 index 504f15d..0000000 --- a/store/state/utils.go +++ /dev/null @@ -1,23 +0,0 @@ -package state - -import ( - uuid "github.com/satori/go.uuid" - "time" -) - -func StringsToMap(list []string) map[string]struct{} { - newMap := make(map[string]struct{}, len(list)) - for _, x := range list { - newMap[x] = struct{}{} - } - return newMap -} - -func GenerateId() string { - uuidV4 := uuid.NewV4() - return uuidV4.String() -} - -func GenerateDate() time.Time { - return time.Now() -} diff --git a/store/state/version_config.go b/store/state/version_config.go deleted file mode 100644 index 960ea8d..0000000 --- a/store/state/version_config.go +++ /dev/null @@ -1,89 +0,0 @@ -package state - -import ( - "github.com/integration-system/isp-lib/v2/config" - "isp-config-service/conf" - "isp-config-service/entity" -) - -const DefaultVersionCount = 15 - -type WriteableVersionConfigStore interface { - ReadonlyVersionConfigStore - Update(config entity.VersionConfig) (removedId string) - Delete(id string) -} - -type ReadonlyVersionConfigStore interface { - GetByConfigId(id string) []entity.VersionConfig -} - -type VersionConfigStore struct { - VersionByConfigId map[string][]entity.VersionConfig -} - -func (s *VersionConfigStore) Update(req entity.VersionConfig) string { - limit := config.Get().(*conf.Configuration).VersionConfigCount - if limit <= 0 { - limit = DefaultVersionCount - } - var removedId string - store, found := s.VersionByConfigId[req.ConfigId] - if found { - if len(store) >= limit { - removedId = store[0].Id - store = append(store[1:], req) - } else { - store = append(store, req) - } - } else { - store = []entity.VersionConfig{req} - } - s.VersionByConfigId[req.ConfigId] = store - return removedId -} - -func (s *VersionConfigStore) Delete(id string) { - for cfgId, versionCfg := range s.VersionByConfigId { - for i, versionConfig := range versionCfg { - if versionConfig.Id == id { - newVersionCfg := versionCfg[:i+copy(versionCfg[i:], versionCfg[i+1:])] - if len(newVersionCfg) == 0 { - delete(s.VersionByConfigId, cfgId) - } else { - s.VersionByConfigId[cfgId] = newVersionCfg - } - return - } - } - } -} - -func (s VersionConfigStore) GetByConfigId(id string) []entity.VersionConfig { - response, found := s.VersionByConfigId[id] - if found { - return s.reverse(response) - } - return []entity.VersionConfig{} -} - -func (s VersionConfigStore) reverse(cfg []entity.VersionConfig) []entity.VersionConfig { - for i, j := 0, len(cfg)-1; i < j; i, j = i+1, j-1 { - cfg[i], cfg[j] = cfg[j], cfg[i] - } - return cfg -} - -func NewVersionConfigStore() *VersionConfigStore { - return &VersionConfigStore{ - VersionByConfigId: make(map[string][]entity.VersionConfig), - } -} - -func NewVersionConfigStoreFromSnapshot(req []entity.VersionConfig) *VersionConfigStore { - store := NewVersionConfigStore() - for i := 0; i < len(req); i++ { - _ = store.Update(req[i]) - } - return store -} diff --git a/store/store.go b/store/store.go deleted file mode 100644 index eab4f63..0000000 --- a/store/store.go +++ /dev/null @@ -1,137 +0,0 @@ -package store - -import ( - "fmt" - "io" - "sync" - - "github.com/hashicorp/raft" - log "github.com/integration-system/isp-log" - jsoniter "github.com/json-iterator/go" - "github.com/mohae/deepcopy" - "github.com/pkg/errors" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/store/state" -) - -var ( - json = jsoniter.ConfigFastest -) - -type Store struct { - state *state.State - lock sync.RWMutex - handlers map[cluster.Command]func([]byte) (interface{}, error) -} - -func (s *Store) Apply(l *raft.Log) interface{} { - s.lock.Lock() - defer s.lock.Unlock() - - if len(l.Data) < cluster.CommandSizeBytes { - log.Errorf(codes.ApplyLogCommandError, "invalid log data command: %s", l.Data) - } - command := cluster.ParseCommand(l.Data) - log.Debugf(0, "Apply %s. Data: %s", command, l.Data) - - var ( - result interface{} - err error - ) - if handler, ok := s.handlers[command]; ok { - result, err = handler(l.Data[8:]) - } else { - err = fmt.Errorf("unknown log command %d", command) - log.WithMetadata(map[string]interface{}{ - "command": command, - "body": string(l.Data), - }).Error(codes.ApplyLogCommandError, "unknown log command") - } - - bytes, e := json.Marshal(result) - if e != nil { - panic(e) // must never occurred - } - - logResponse := cluster.ApplyLogResponse{Result: bytes} - if err != nil { - logResponse.ApplyError = err.Error() - } - return logResponse -} - -func (s *Store) Snapshot() (raft.FSMSnapshot, error) { - s.lock.Lock() - copied := deepcopy.Copy(s.state).(*state.State) - s.lock.Unlock() - return &fsmSnapshot{copied}, nil -} - -func (s *Store) Restore(rc io.ReadCloser) error { - state2 := state.State{} - if err := json.NewDecoder(rc).Decode(&state2); err != nil { - return errors.WithMessage(err, "unmarshal store") - } - s.state = &state2 - return nil -} - -func (s *Store) VisitReadonlyState(f func(state.ReadonlyState)) { - s.lock.RLock() - defer s.lock.RUnlock() - - f(*s.state) -} - -func (s *Store) VisitState(f func(writableState state.WritableState)) { - s.lock.Lock() - defer s.lock.Unlock() - f(s.state) -} - -type fsmSnapshot struct { - state *state.State -} - -func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error { - err := func() error { - b, err := json.Marshal(f.state) - if err != nil { - return err - } - if _, err := sink.Write(b); err != nil { - return err - } - return sink.Close() - }() - - if err != nil { - _ = sink.Cancel() - } - return err -} - -func (f *fsmSnapshot) Release() {} - -func NewStateStore(st *state.State) *Store { - store := &Store{ - state: st, - } - store.handlers = map[cluster.Command]func([]byte) (interface{}, error){ - cluster.UpdateBackendDeclarationCommand: store.applyUpdateBackendDeclarationCommand, - cluster.DeleteBackendDeclarationCommand: store.applyDeleteBackendDeclarationCommand, - cluster.ModuleConnectedCommand: store.applyModuleConnectedCommand, - cluster.ModuleDisconnectedCommand: store.applyModuleDisconnectedCommand, - cluster.DeleteModulesCommand: store.applyDeleteModulesCommand, - cluster.UpdateConfigSchemaCommand: store.applyUpdateConfigSchemaCommand, - cluster.ActivateConfigCommand: store.applyActivateConfigCommand, - cluster.DeleteConfigsCommand: store.applyDeleteConfigsCommand, - cluster.UpsertConfigCommand: store.applyUpsertConfigCommand, - cluster.DeleteCommonConfigsCommand: store.applyDeleteCommonConfigsCommand, - cluster.UpsertCommonConfigCommand: store.applyUpsertCommonConfigCommand, - cluster.BroadcastEventCommand: store.applyBroadcastEventCommand, - cluster.DeleteVersionConfigCommand: store.applyDeleteVersionConfigCommand, - } - return store -} diff --git a/subs/client.go b/subs/client.go deleted file mode 100644 index 635b3a6..0000000 --- a/subs/client.go +++ /dev/null @@ -1,124 +0,0 @@ -package subs - -import ( - "fmt" - "time" - - "github.com/asaskevich/govalidator" - etp "github.com/integration-system/isp-etp-go/v2" - "github.com/integration-system/isp-lib/v2/bootstrap" - schema2 "github.com/integration-system/isp-lib/v2/config/schema" - "github.com/integration-system/isp-lib/v2/structure" - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - "isp-config-service/cluster" - "isp-config-service/entity" - "isp-config-service/service" - "isp-config-service/store/state" -) - -func (h *SocketEventHandler) handleModuleReady(conn etp.Conn, data []byte) []byte { - moduleName, _ := Parameters(conn) // REMOVE - log.Debugf(0, "handleModuleReady moduleName: %s", moduleName) // REMOVE - declaration := structure.BackendDeclaration{} - err := json.Unmarshal(data, &declaration) - if err != nil { - return []byte(err.Error()) - } - - _, err = govalidator.ValidateStruct(declaration) - if err != nil { - return []byte(err.Error()) - } - SetBackendDeclaration(conn, declaration) - command := cluster.PrepareUpdateBackendDeclarationCommand(declaration) - _, err = SyncApplyCommand(command) - if err != nil { - return []byte(err.Error()) - } - return []byte(utils.WsOkResponse) -} - -func (h *SocketEventHandler) handleModuleRequirements(conn etp.Conn, data []byte) []byte { - moduleName, err := Parameters(conn) - log.Debugf(0, "handleModuleRequirements moduleName: %s", moduleName) // REMOVE - if err != nil { - return []byte(err.Error()) - } - - declaration := bootstrap.ModuleRequirements{} - err = json.Unmarshal(data, &declaration) - if err != nil { - return []byte(err.Error()) - } - - h.store.VisitReadonlyState(func(state state.ReadonlyState) { - service.Discovery.Subscribe(conn, declaration.RequiredModules, state.Mesh()) - if declaration.RequireRoutes { - service.Routes.SubscribeRoutes(conn, state.Mesh()) - } - }) - return []byte(utils.WsOkResponse) -} - -func (h *SocketEventHandler) handleConfigSchema(conn etp.Conn, data []byte) []byte { - moduleName, err := Parameters(conn) - log.Debugf(0, "handleConfigSchema moduleName: %s", moduleName) // REMOVE - if err != nil { - return []byte(err.Error()) - } - - configSchema := schema2.ConfigSchema{} - if err := json.Unmarshal(data, &configSchema); err != nil { - return []byte(err.Error()) - } - - module := new(entity.Module) - h.store.VisitReadonlyState(func(readState state.ReadonlyState) { - module = readState.Modules().GetByName(moduleName) - }) - if module == nil { - return []byte(fmt.Sprintf("module with name %s not found", moduleName)) - } - - now := state.GenerateDate() - schema := entity.ConfigSchema{ - Id: state.GenerateId(), - Version: configSchema.Version, - ModuleId: module.Id, - Schema: configSchema.Schema, - CreatedAt: now, - UpdatedAt: now, - } - command := cluster.PrepareUpdateConfigSchemaCommand(schema) - _, _ = SyncApplyCommand(command) - - var configs []entity.Config - h.store.VisitReadonlyState(func(readState state.ReadonlyState) { - configs = readState.Configs().GetByModuleIds([]string{module.Id}) - }) - if len(configs) == 0 { // create empty default config - if configSchema.DefaultConfig == nil { - configSchema.DefaultConfig = make(map[string]interface{}) - } - config := entity.Config{ - Id: state.GenerateId(), - Name: module.Name, - ModuleId: module.Id, - Data: configSchema.DefaultConfig, - CreatedAt: now, - UpdatedAt: now, - } - upsertConfig := cluster.UpsertConfig{ - Config: config, - Create: true, - Unsafe: true, - VersionId: state.GenerateId(), - VersionCreateAt: time.Now(), - } - - command := cluster.PrepareUpsertConfigCommand(upsertConfig) - _, _ = SyncApplyCommand(command) - } - return []byte(utils.WsOkResponse) -} diff --git a/subs/cluster.go b/subs/cluster.go deleted file mode 100644 index 696e1a1..0000000 --- a/subs/cluster.go +++ /dev/null @@ -1,31 +0,0 @@ -package subs - -import ( - "fmt" - - etp "github.com/integration-system/isp-etp-go/v2" - log "github.com/integration-system/isp-log" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/holder" -) - -func (h *SocketEventHandler) applyCommandOnLeader(_ etp.Conn, cmd []byte) (data []byte) { - cmdCopy := make([]byte, len(cmd)) - copy(cmdCopy, cmd) - obj, err := holder.ClusterClient.SyncApplyOnLeader(cmdCopy) - if err != nil { - var logResponse cluster.ApplyLogResponse - logResponse.ApplyError = fmt.Sprintf("SyncApplyOnLeader: %v", err) - data, err = json.Marshal(logResponse) - if err != nil { - log.Fatalf(codes.SyncApplyError, "marshaling ApplyLogResponse: %v", err) - } - return data - } - data, err = json.Marshal(obj) - if err != nil { - log.Fatalf(codes.SyncApplyError, "marshaling ApplyLogResponse: %v", err) - } - return data -} diff --git a/subs/conn.go b/subs/conn.go deleted file mode 100644 index 33fd02c..0000000 --- a/subs/conn.go +++ /dev/null @@ -1,31 +0,0 @@ -package subs - -import ( - etp "github.com/integration-system/isp-etp-go/v2" - "github.com/integration-system/isp-lib/v2/structure" - "github.com/integration-system/isp-lib/v2/utils" - "isp-config-service/cluster" -) - -type connectionState struct { - declaration structure.BackendDeclaration -} - -func IsConfigClusterNode(conn etp.Conn) bool { - isCluster := conn.URL().Query().Get(cluster.GETParam) - return isCluster == "true" -} - -func Parameters(conn etp.Conn) (moduleName string, err error) { - _, moduleName, err = utils.ParseParameters(conn.URL().RawQuery) - return moduleName, err -} - -func SetBackendDeclaration(conn etp.Conn, backend structure.BackendDeclaration) { - conn.SetContext(connectionState{declaration: backend}) -} - -func ExtractBackendDeclaration(conn etp.Conn) (structure.BackendDeclaration, bool) { - declaration, ok := conn.Context().(connectionState) - return declaration.declaration, ok -} diff --git a/subs/helpers.go b/subs/helpers.go deleted file mode 100644 index db91fb5..0000000 --- a/subs/helpers.go +++ /dev/null @@ -1,51 +0,0 @@ -package subs - -import ( - "errors" - - etp "github.com/integration-system/isp-etp-go/v2" - log "github.com/integration-system/isp-log" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/holder" - "isp-config-service/service" -) - -func EmitConn(conn etp.Conn, event string, body []byte) { - err := service.EmitConnWithTimeout(conn, event, body) - if err != nil { - log.WithMetadata(map[string]interface{}{ - "event": event, - }).Warnf(codes.WebsocketEmitError, "emit err %v", err) - } -} - -func SyncApplyCommand(command []byte) (interface{}, error) { - applyLogResponse, err := holder.ClusterClient.SyncApply(command) - if err != nil { - commandName := cluster.ParseCommand(command).String() - log.WithMetadata(map[string]interface{}{ - "command": string(command), - "commandName": commandName, - }).Warnf(codes.SyncApplyError, "apply command: %v", err) - return nil, err - } - if applyLogResponse != nil && applyLogResponse.ApplyError != "" { - commandName := cluster.ParseCommand(command).String() - log.WithMetadata(map[string]interface{}{ - "result": string(applyLogResponse.Result), - "applyError": applyLogResponse.ApplyError, - "commandName": commandName, - }).Warnf(codes.SyncApplyError, "apply command") - return applyLogResponse.Result, errors.New(applyLogResponse.ApplyError) - } - return applyLogResponse.Result, nil -} - -func FormatErrorConnection(err error) []byte { - errMap := map[string]interface{}{ - "error": err.Error(), - } - data, _ := json.Marshal(errMap) - return data -} diff --git a/subs/root.go b/subs/root.go deleted file mode 100644 index 30f4af7..0000000 --- a/subs/root.go +++ /dev/null @@ -1,118 +0,0 @@ -package subs - -import ( - etp "github.com/integration-system/isp-etp-go/v2" - "github.com/integration-system/isp-lib/v2/utils" - log "github.com/integration-system/isp-log" - jsoniter "github.com/json-iterator/go" - "isp-config-service/cluster" - "isp-config-service/codes" - "isp-config-service/entity" - "isp-config-service/holder" - "isp-config-service/service" - "isp-config-service/store" - "isp-config-service/store/state" -) - -var ( - json = jsoniter.ConfigFastest -) - -type SocketEventHandler struct { - server etp.Server - store *store.Store -} - -func (h *SocketEventHandler) SubscribeAll() { - h.server. - OnConnect(h.handleConnect). - OnDisconnect(h.handleDisconnect). - OnError(h.handleError). - OnWithAck(cluster.ApplyCommandEvent, h.applyCommandOnLeader). - OnWithAck(utils.ModuleReady, h.handleModuleReady). - OnWithAck(utils.ModuleSendRequirements, h.handleModuleRequirements). - OnWithAck(utils.ModuleSendConfigSchema, h.handleConfigSchema) -} - -func (h *SocketEventHandler) handleConnect(conn etp.Conn) { - isClusterNode := IsConfigClusterNode(conn) - moduleName, err := Parameters(conn) - log.Debugf(0, "handleConnect from %s", moduleName) // REMOVE - if err != nil { - EmitConn(conn, utils.ErrorConnection, FormatErrorConnection(err)) - _ = conn.Close() - return - } - holder.EtpServer.Rooms().Join(conn, service.Room.Module(moduleName)) - now := state.GenerateDate() - module := entity.Module{ - Id: state.GenerateId(), - Name: moduleName, - CreatedAt: now, - LastConnectedAt: now, - } - command := cluster.PrepareModuleConnectedCommand(module) - _, err = SyncApplyCommand(command) - if err != nil && !isClusterNode { - EmitConn(conn, utils.ErrorConnection, FormatErrorConnection(err)) - _ = conn.Close() - return - } - - if isClusterNode { - return - } - - var config map[string]interface{} - h.store.VisitReadonlyState(func(state state.ReadonlyState) { - config, err = service.ConfigService.GetCompiledConfig(moduleName, state) - }) - if err != nil { - EmitConn(conn, utils.ConfigError, []byte(err.Error())) - return - } - data, err := json.Marshal(config) - if err != nil { - EmitConn(conn, utils.ConfigError, []byte(err.Error())) - return - } - EmitConn(conn, utils.ConfigSendConfigWhenConnected, data) -} - -func (h *SocketEventHandler) handleDisconnect(conn etp.Conn, _ error) { - moduleName, _ := Parameters(conn) - log.Debugf(0, "websocket handleDisconnect from module '%s' addr '%s'", moduleName, conn.RemoteAddr()) - if moduleName != "" { - holder.EtpServer.Rooms().Leave(conn, service.Room.Module(moduleName)) - now := state.GenerateDate() - module := entity.Module{ - Id: state.GenerateId(), - Name: moduleName, - CreatedAt: now, - LastConnectedAt: now, - LastDisconnectedAt: now, - } - command := cluster.PrepareModuleDisconnectedCommand(module) - _, _ = SyncApplyCommand(command) - } - - service.Discovery.HandleDisconnect(conn.ID()) - service.Routes.HandleDisconnect(conn.ID()) - backend, ok := ExtractBackendDeclaration(conn) - if ok { - command := cluster.PrepareDeleteBackendDeclarationCommand(backend) - _, _ = SyncApplyCommand(command) - } -} - -func (h *SocketEventHandler) handleError(conn etp.Conn, err error) { - moduleName, _ := Parameters(conn) - log.Warnf(codes.WebsocketError, "websocket handleError module '%s' from %s: %v", moduleName, conn.RemoteAddr(), err) -} - -func NewSocketEventHandler(server etp.Server, store *store.Store) *SocketEventHandler { - return &SocketEventHandler{ - server: server, - store: store, - } -} From 2e23e882bb87675332c9a0e15a20edb956248d2f Mon Sep 17 00:00:00 2001 From: d1slike Date: Fri, 3 May 2024 18:24:07 +0300 Subject: [PATCH 02/40] Implement lifecycle --- assembly/locator.go | 12 ++--- main.go | 2 +- migrations/20240503120009_init.sql | 3 ++ service/startup/service.go | 80 ++++++++++++++++++++++++++---- 4 files changed, 79 insertions(+), 18 deletions(-) diff --git a/assembly/locator.go b/assembly/locator.go index 55e9d26..325304b 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -28,9 +28,9 @@ func NewLocator(logger log.Logger, db db.DB) Locator { } type Config struct { - grpcMux *grpc.Mux - httpMux http.Handler - etpSrv *etp.Server + GrpcMux *grpc.Mux + HttpMux http.Handler + EtpSrv *etp.Server } func (l Locator) Config() Config { @@ -49,8 +49,8 @@ func (l Locator) Config() Config { httpMux := routes.HttpHandler(etpSrv) return Config{ - grpcMux: grpcMux, - etpSrv: etpSrv, - httpMux: httpMux, + GrpcMux: grpcMux, + EtpSrv: etpSrv, + HttpMux: httpMux, } } diff --git a/main.go b/main.go index caa7d2f..8e77fa4 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,7 @@ func main() { startup := startup.New(boot) app.AddRunners(startup) - app.AddClosers(startup) + app.AddClosers(startup.Closers()...) shutdown.On(func() { logger.Info(app.Context(), "starting shutdown") diff --git a/migrations/20240503120009_init.sql b/migrations/20240503120009_init.sql index 58befe1..f821fbf 100644 --- a/migrations/20240503120009_init.sql +++ b/migrations/20240503120009_init.sql @@ -53,3 +53,6 @@ create index IX_isp_config_service__config_schema__module_id on isp_config_servi -- +goose Down drop table isp_config_service__module; +drop table isp_config_service__config; +drop table isp_config_service__config_history; +drop table isp_config_service__config_schema; diff --git a/service/startup/service.go b/service/startup/service.go index a4ee2a3..e673540 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -7,11 +7,16 @@ import ( "github.com/pkg/errors" "github.com/pressly/goose/v3" + "github.com/txix-open/etp/v3" + "github.com/txix-open/isp-kit/app" "github.com/txix-open/isp-kit/bootstrap" "github.com/txix-open/isp-kit/dbx/migration" + "github.com/txix-open/isp-kit/grpc" + "github.com/txix-open/isp-kit/http" "github.com/txix-open/isp-kit/http/httpclix" "github.com/txix-open/isp-kit/log" "isp-config-service/assembly" + "isp-config-service/conf" "isp-config-service/service/rqlite" "isp-config-service/service/rqlite/db" "isp-config-service/service/rqlite/goose_store" @@ -22,22 +27,36 @@ const ( ) type Service struct { - boot *bootstrap.Bootstrap - logger log.Logger - rqlite *rqlite.Rqlite + boot *bootstrap.Bootstrap + rqlite *rqlite.Rqlite + grpcSrv *grpc.Server + httpSrv *http.Server + logger log.Logger + + //initialized in Run + etpSrv *etp.Server } func New(boot *bootstrap.Bootstrap) Service { rqlite := rqlite.New(boot.App.Config()) return Service{ - boot: boot, - logger: boot.App.Logger(), - rqlite: rqlite, + boot: boot, + rqlite: rqlite, + logger: boot.App.Logger(), + httpSrv: http.NewServer(boot.App.Logger()), + grpcSrv: grpc.DefaultServer(), } } func (s Service) Run(ctx context.Context) error { + localConfig := conf.Local{} + err := s.boot.App.Config().Read(&localConfig) + if err != nil { + return errors.WithMessage(err, "read local config") + } + go func() { + s.logger.Debug(ctx, "running embedded rqlite...") err := s.rqlite.Run(ctx) if err != nil { s.boot.Fatal(errors.WithMessage(err, "run embedded rqlite")) @@ -46,7 +65,7 @@ func (s Service) Run(ctx context.Context) error { time.Sleep(1 * time.Second) //optimistically wait for store initialization s.logger.Debug(ctx, fmt.Sprintf("waiting for cluster startup for %s...", waitForLeaderTimeout)) - err := s.rqlite.WaitForLeader(waitForLeaderTimeout) + err = s.rqlite.WaitForLeader(waitForLeaderTimeout) if err != nil { return errors.WithMessage(err, "wait for leader") } @@ -66,14 +85,53 @@ func (s Service) Run(ctx context.Context) error { return errors.WithMessage(err, "dial to embedded rqlite") } - _ = assembly.NewLocator(s.logger, db) + config := assembly.NewLocator(s.logger, db).Config() + s.etpSrv = config.EtpSrv + s.grpcSrv.Upgrade(config.GrpcMux) + s.httpSrv.Upgrade(config.HttpMux) + + go func() { + s.logger.Debug(ctx, fmt.Sprintf("starting grpc server on %s", s.boot.BindingAddress)) + err := s.grpcSrv.ListenAndServe(s.boot.BindingAddress) + if err != nil { + s.boot.Fatal(errors.WithMessage(err, "start grpc server")) + } + }() + + go func() { + httpPort := localConfig.ConfigServiceAddress.Port + s.logger.Debug(ctx, fmt.Sprintf("starting http server on 0.0.0.0:%s", httpPort)) + err := s.httpSrv.ListenAndServe(fmt.Sprintf("0.0.0.0:%s", httpPort)) + if err != nil { + s.boot.Fatal(errors.WithMessage(err, "start http server")) + } + }() + time.Sleep(1 * time.Second) return nil } -func (s Service) Close() error { - _ = s.rqlite.Close() - return nil +func (s Service) Closers() []app.Closer { + return []app.Closer{ + app.CloserFunc(func() error { + s.grpcSrv.Shutdown() + return nil + }), + app.CloserFunc(func() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return s.httpSrv.Shutdown(ctx) + }), + app.CloserFunc(func() error { + if s.etpSrv == nil { + return nil + } + s.etpSrv.OnDisconnect(nil) + s.etpSrv.Shutdown() + return nil + }), + s.rqlite, + } } func (s Service) leaderStartup(ctx context.Context) error { From 8a2a65d8ed32d34e6489848e22f364532f10cfb8 Mon Sep 17 00:00:00 2001 From: d1slike Date: Mon, 6 May 2024 09:43:11 +0300 Subject: [PATCH 03/40] Implement lifecycle --- controller/module.go | 108 +++++++++++++++++++-- entity/backend.go | 16 ++++ entity/config.go | 34 +++++++ entity/json.go | 45 +++++++++ entity/module.go | 10 +- go.mod | 8 +- migrations/20240503120009_init.sql | 28 ++++-- repository/backend.go | 64 +++++++++++++ repository/module.go | 46 ++++++++- service/module.go | 149 +++++++++++++++++++++++++++++ service/rqlite/db/consistency.go | 82 ++++++++++++++++ service/rqlite/db/db.go | 31 ++++-- service/rqlite/db/request.go | 6 +- service/startup/service.go | 3 +- 14 files changed, 593 insertions(+), 37 deletions(-) create mode 100644 entity/backend.go create mode 100644 entity/config.go create mode 100644 entity/json.go create mode 100644 repository/backend.go create mode 100644 service/rqlite/db/consistency.go diff --git a/controller/module.go b/controller/module.go index 146b5e8..1a1d7d0 100644 --- a/controller/module.go +++ b/controller/module.go @@ -3,37 +3,127 @@ package controller import ( "context" + "github.com/pkg/errors" "github.com/txix-open/etp/v3" "github.com/txix-open/etp/v3/msg" + "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/json" + "github.com/txix-open/isp-kit/log" ) +var ( + ok = []byte("ok") +) + +type ModuleService interface { + OnConnect(conn *etp.Conn, moduleName string) error + OnDisconnect(conn *etp.Conn, moduleName string, isNormalClose bool, err error) error + OnError(conn *etp.Conn, moduleName string, err error) + OnModuleReady( + ctx context.Context, + conn *etp.Conn, + backend cluster.BackendDeclaration, + ) error + OnModuleRequirements( + ctx context.Context, + conn *etp.Conn, + requirements cluster.ModuleRequirements, + ) error + OnModuleConfigSchema( + ctx context.Context, + conn *etp.Conn, + data cluster.ConfigData, + ) error +} + type Module struct { + service ModuleService + logger log.Logger } -func NewModule() Module { - return Module{} +func NewModule(service ModuleService, logger log.Logger) Module { + return Module{ + service: service, + logger: logger, + } } func (m Module) OnConnect(conn *etp.Conn) { - + err := m.service.OnConnect(conn, moduleName(conn)) + if err != nil { + m.handleError(context.Background(), err) + } } -func (m Module) OnDisconnect(ctx *etp.Conn, conn error) { - +func (m Module) OnDisconnect(conn *etp.Conn, err error) { + handleDisconnectErr := m.service.OnDisconnect( + conn, + moduleName(conn), + etp.IsNormalClose(err), + err, + ) + if handleDisconnectErr != nil { + m.handleError(context.Background(), handleDisconnectErr) + } } func (m Module) OnError(conn *etp.Conn, err error) { - + m.service.OnError(conn, moduleName(conn), err) } func (m Module) OnModuleReady(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { - return nil + backend := cluster.BackendDeclaration{} + err := json.Unmarshal(event.Data, &backend) + if err != nil { + return m.handleError(ctx, errors.WithMessage(err, "unmarshal event data")) + } + + err = m.service.OnModuleReady(ctx, conn, backend) + if err != nil { + return m.handleError(ctx, err) + } + + return ok } func (m Module) OnModuleRequirements(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { - return nil + requirements := cluster.ModuleRequirements{} + err := json.Unmarshal(event.Data, &requirements) + if err != nil { + return m.handleError(ctx, errors.WithMessage(err, "unmarshal event data")) + } + + err = m.service.OnModuleRequirements(ctx, conn, requirements) + if err != nil { + return m.handleError(ctx, err) + } + + return ok } func (m Module) OnModuleConfigSchema(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { - return nil + configData := cluster.ConfigData{} + err := json.Unmarshal(event.Data, &configData) + if err != nil { + return m.handleError(ctx, errors.WithMessage(err, "unmarshal event data")) + } + + err = m.service.OnModuleConfigSchema(ctx, conn, configData) + if err != nil { + return m.handleError(ctx, err) + } + + return ok +} + +func (m Module) handleError( + ctx context.Context, + err error, +) []byte { + m.logger.Error(ctx, err) + return []byte(err.Error()) +} + +func moduleName(conn *etp.Conn) string { + return conn.HttpRequest().Form.Get("module_name") } diff --git a/entity/backend.go b/entity/backend.go new file mode 100644 index 0000000..e7df19a --- /dev/null +++ b/entity/backend.go @@ -0,0 +1,16 @@ +package entity + +import ( + "github.com/txix-open/isp-kit/cluster" +) + +type Backend struct { + ModuleId string `json:"module_id"` + Address string `json:"address"` + Version string `json:"version"` + LibVersion string `json:"lib_version"` + Endpoints JsonValue[[]cluster.EndpointDescriptor] `json:"endpoints"` + RequiredModules JsonValue[cluster.ModuleRequirements] `json:"required_modules"` + //CreatedAt time.Time `json:"created_at"` + //UpdatedAt time.Time `json:"updated_at"` +} diff --git a/entity/config.go b/entity/config.go new file mode 100644 index 0000000..fd36826 --- /dev/null +++ b/entity/config.go @@ -0,0 +1,34 @@ +package entity + +import ( + "time" +) + +type Config struct { + Id string `json:"id"` + Name string `json:"name"` + ModuleId string `json:"module_id"` + Data []byte `json:"data"` + Version int `json:"version"` + Active int `json:"active"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type ConfigHistory struct { + Id string `json:"id"` + ConfigId string `json:"config_id"` + Data []byte `json:"data"` + Version int `json:"version"` + AdminId int `json:"admin_id"` + CreatedAt time.Time `json:"created_at"` +} + +type ConfigSchema struct { + Id string `json:"id"` + ModuleId string `json:"module_id"` + Data []byte `json:"data"` + ModuleVersion string `json:"module_version"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/entity/json.go b/entity/json.go new file mode 100644 index 0000000..4165bdb --- /dev/null +++ b/entity/json.go @@ -0,0 +1,45 @@ +package entity + +import ( + "encoding/base64" + "strconv" + + "github.com/txix-open/isp-kit/json" +) + +type JsonValue[T any] struct { + Value T +} + +func (l JsonValue[T]) MarshalJSON() ([]byte, error) { + data, err := l.MarshalText() + if err != nil { + return nil, err + } + return []byte(strconv.Quote(string(data))), nil +} + +func (l *JsonValue[T]) UnmarshalJSON(data []byte) error { + s, err := strconv.Unquote(string(data)) + if err != nil { + return err + } + return l.UnmarshalText([]byte(s)) +} + +func (l JsonValue[T]) MarshalText() ([]byte, error) { + data, err := json.Marshal(l.Value) + if err != nil { + return nil, err + } + s := base64.StdEncoding.EncodeToString(data) + return []byte(s), nil +} + +func (l *JsonValue[T]) UnmarshalText(data []byte) error { + arr, err := base64.StdEncoding.DecodeString(string(data)) + if err != nil { + return err + } + return json.Unmarshal(arr, &l.Value) +} diff --git a/entity/module.go b/entity/module.go index 49ac4c7..5d1836f 100644 --- a/entity/module.go +++ b/entity/module.go @@ -5,9 +5,9 @@ import ( ) type Module struct { - Id string - Name string - LastConnectedAt time.Time - LastDisconnectedAt time.Time - Created time.Time + Id string `json:"id"` + Name string `json:"name"` + LastConnectedAt *time.Time `json:"last_connected_at"` + LastDisconnectedAt *time.Time `json:"last_disconnected_at"` + CreatedAt time.Time `json:"created_at"` } diff --git a/go.mod b/go.mod index a9e5c42..fb9cddf 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,16 @@ module isp-config-service go 1.22 require ( + github.com/Masterminds/squirrel v1.5.4 + github.com/google/uuid v1.6.0 + github.com/iancoleman/strcase v0.3.0 github.com/jmoiron/sqlx v1.4.0 github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.20.0 github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 github.com/rqlite/rqlite/v8 v8.23.4 github.com/tidwall/gjson v1.17.1 - github.com/txix-open/etp/v3 v3.0.0 + github.com/txix-open/etp/v3 v3.1.0 github.com/txix-open/isp-kit v1.31.2 ) @@ -36,12 +39,13 @@ require ( github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/raft v1.6.1 // indirect - github.com/iancoleman/strcase v0.3.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/migrations/20240503120009_init.sql b/migrations/20240503120009_init.sql index f821fbf..1fbddd1 100644 --- a/migrations/20240503120009_init.sql +++ b/migrations/20240503120009_init.sql @@ -39,20 +39,32 @@ create index IX_isp_config_service__config_history__config_id on isp_config_serv create table isp_config_service__config_schema ( - id text not null primary key, - name text not null, - module_id text not null, - data blob not null, - version text not null default 1, - created_at text not null default (datetime('now')), - updated_at text not null default (datetime('now')), + id text not null primary key, + module_id text not null unique, + data blob not null, + module_version text not null default 1, + created_at text not null default (datetime('now')), + updated_at text not null default (datetime('now')), foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade ); -create index IX_isp_config_service__config_schema__module_id on isp_config_service__config_schema (module_id); +create table isp_config_service__backend +( + module_id text not null, + address text not null, + version text not null, + lib_version text not null, + endpoints blob not null, + required_modules blob not null, + created_at text not null default (datetime('now')), + updated_at text not null default (datetime('now')), + primary key (module_id, address), + foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade +); -- +goose Down drop table isp_config_service__module; drop table isp_config_service__config; drop table isp_config_service__config_history; drop table isp_config_service__config_schema; +drop table isp_config_service__backend; diff --git a/repository/backend.go b/repository/backend.go new file mode 100644 index 0000000..c589f59 --- /dev/null +++ b/repository/backend.go @@ -0,0 +1,64 @@ +package repository + +import ( + "context" + + "github.com/Masterminds/squirrel" + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/db" + "isp-config-service/entity" +) + +type Backend struct { + db db.DB +} + +func NewBackend(db db.DB) Backend { + return Backend{ + db: db, + } +} + +func (r Backend) Upsert(ctx context.Context, backend entity.Backend) error { + query, args, err := squirrel.Insert(Table("backend")). + Columns("module_id", "address", + "version", "lib_version", + "endpoints", "required_modules"). + Values(backend.ModuleId, backend.Address, + backend.Version, backend.LibVersion, + backend.Endpoints, backend.RequiredModules, + ).Suffix(`on conflict (module_id, address) do update + set version = excluded.version, lib_version = excluded.lib_version, + endpoints = excluded.endpoints, required_modules = excluded.required_modules, + updated_at = datetime('now') + `). + ToSql() + if err != nil { + return errors.WithMessage(err, "build query") + } + + _, err = r.db.Exec(ctx, query, args...) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + + return nil +} + +func (r Backend) Delete(ctx context.Context, moduleId string, address string) error { + query, args, err := squirrel.Delete(Table("backend")). + Where(squirrel.Eq{ + "module_id": moduleId, + "address": address, + }).ToSql() + if err != nil { + return errors.WithMessage(err, "build query") + } + + _, err = r.db.Exec(ctx, query, args...) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + + return nil +} diff --git a/repository/module.go b/repository/module.go index 22a1f57..462054f 100644 --- a/repository/module.go +++ b/repository/module.go @@ -2,7 +2,11 @@ package repository import ( "context" + "fmt" + "time" + "github.com/Masterminds/squirrel" + "github.com/pkg/errors" "github.com/txix-open/isp-kit/db" "isp-config-service/entity" ) @@ -17,6 +21,46 @@ func NewModule(db db.DB) Module { } } -func (r Module) Upsert(ctx context.Context, module entity.Module) error { +func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error) { + query, args, err := squirrel.Insert(Table("module")). + Columns("id", "name", "last_connected_at"). + Values(module.Id, module.Name, module.LastConnectedAt). + Suffix("on conflict (name) do update set last_connected_at = excluded.last_connected_at"). + Suffix("returning id"). + ToSql() + if err != nil { + return "", errors.WithMessage(err, "build query") + } + + result := make(map[string]string) + err = r.db.SelectRow(ctx, &result, query, args...) + if err != nil { + return "", errors.WithMessagef(err, "select : %s", query) + } + return result["id"], nil +} + +func (r Module) SetDisconnectedAt( + ctx context.Context, + moduleId string, + disconnected time.Time, +) error { + query, args, err := squirrel.Update(Table("module")). + Set("last_disconnected_at", disconnected). + Where(squirrel.Eq{"id": moduleId}). + ToSql() + if err != nil { + return errors.WithMessage(err, "build query") + } + + _, err = r.db.Exec(ctx, query, args...) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + + return nil +} +func Table(table string) string { + return fmt.Sprintf("isp_config_service__%s", table) } diff --git a/service/module.go b/service/module.go index 73c93da..8fe9c3f 100644 --- a/service/module.go +++ b/service/module.go @@ -1,4 +1,153 @@ package service +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/txix-open/etp/v3" + "github.com/txix-open/etp/v3/store" + "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/log" + "isp-config-service/entity" +) + +const ( + moduleIdKey = "moduleId" + backendKey = "backend" +) + +type ModuleRepo interface { + Upsert(ctx context.Context, module entity.Module) (string, error) + SetDisconnectedAt( + ctx context.Context, + moduleId string, + disconnected time.Time, + ) error +} + +type BackendRepo interface { + Upsert(ctx context.Context, backend entity.Backend) error + Delete(ctx context.Context, moduleId string, address string) error +} + type Module struct { + moduleRepo ModuleRepo + backendRepo BackendRepo + logger log.Logger +} + +func NewModule( + moduleRepo ModuleRepo, + backendRepo BackendRepo, + logger log.Logger, +) Module { + return Module{ + moduleRepo: moduleRepo, + backendRepo: backendRepo, + logger: logger, + } +} + +func (s Module) OnConnect(conn *etp.Conn, moduleName string) error { + now := time.Now().UTC() + module := entity.Module{ + Id: uuid.NewString(), + Name: moduleName, + LastConnectedAt: &now, + CreatedAt: now, + } + moduleId, err := s.moduleRepo.Upsert(context.Background(), module) + if err != nil { + return errors.WithMessage(err, "upsert module in store") + } + + conn.Data().Set(moduleIdKey, moduleId) + + return nil +} + +func (s Module) OnDisconnect( + conn *etp.Conn, + moduleName string, + isNormalClose bool, + err error, +) error { + if isNormalClose { + s.logger.Info(context.Background(), fmt.Sprintf("module '%s' disconnected", moduleName)) + } else { + message := errors.WithMessagef( + err, + "module '%s' unexpectedly disconnected", + moduleName, + ) + s.logger.Error(context.Background(), message) + } + + moduleId, _ := store.Get[string](conn.Data(), moduleIdKey) + if moduleId != "" { + err = s.moduleRepo.SetDisconnectedAt(context.Background(), moduleName, time.Now().UTC()) + if err != nil { + return errors.WithMessage(err, "update disconnected time in store") + } + } + + backend, _ := store.Get[entity.Backend](conn.Data(), backendKey) + if backend.ModuleId != "" { + err = s.backendRepo.Delete(context.Background(), backend.ModuleId, backend.Address) + if err != nil { + return errors.WithMessage(err, "delete backend in store") + } + } + + return nil +} + +func (s Module) OnError(conn *etp.Conn, moduleName string, err error) { + err = errors.WithMessagef( + err, + "unexpected error in communication, module: '%s'", + moduleName, + ) + s.logger.Error(context.Background(), err) +} + +func (s Module) OnModuleReady( + ctx context.Context, + conn *etp.Conn, + backend cluster.BackendDeclaration, +) error { + moduleId, err := store.Get[string](conn.Data(), moduleIdKey) + if err != nil { + return errors.WithMessage(err, "resolve module id") + } + + backend := entity.Backend{ + ModuleId: moduleId, + Address: fmt.Sprintf("%s:%s", backend.Address.IP, backend.Address.Port), + Version: backend.Version, + LibVersion: backend.LibVersion, + Endpoints: nil, + RequiredModules: nil, + CreatedAt: time.Time{}, + UpdatedAt: time.Time{}, + } +} + +func (s Module) OnModuleRequirements( + ctx context.Context, + conn *etp.Conn, + requirements cluster.ModuleRequirements, +) error { + return nil +} + +func (s Module) OnModuleConfigSchema( + ctx context.Context, + conn *etp.Conn, + data cluster.ConfigData, +) error { + return nil } diff --git a/service/rqlite/db/consistency.go b/service/rqlite/db/consistency.go new file mode 100644 index 0000000..510373b --- /dev/null +++ b/service/rqlite/db/consistency.go @@ -0,0 +1,82 @@ +package db + +import ( + "context" + "time" +) + +type consistencyCtxKey struct{} + +var ( + consistencyCtxValue = consistencyCtxKey{} +) + +func ConsistencyFromContext(ctx context.Context) Consistency { + c, ok := ctx.Value(consistencyCtxValue).(Consistency) + if !ok { + return Consistency{} + } + return c +} + +type ConsistencyLevel string + +const ( + ConsistencyLevelNone ConsistencyLevel = "none" + ConsistencyLevelWeak ConsistencyLevel = "weak" + ConsistencyLevelStrong ConsistencyLevel = "strong" +) + +type Consistency struct { + level ConsistencyLevel + freshness time.Duration + freshnessStrict bool +} + +func (c Consistency) WithLevel(level ConsistencyLevel) Consistency { + c.level = level + return c +} + +func (c Consistency) WithFreshness(freshness time.Duration) Consistency { + c.freshness = freshness + return c +} + +func (c Consistency) WithFreshnessStrict(freshnessStrict bool) Consistency { + c.freshnessStrict = freshnessStrict + return c +} +func (c Consistency) ToContext(ctx context.Context) context.Context { + return context.WithValue(ctx, consistencyCtxValue, c) +} + +func (c Consistency) appendParams(params map[string]any) { + if c.level != "" { + params["level"] = c.level + } + if c.freshness > 0 { + params["freshness"] = c.freshness.String() + } + if c.freshnessStrict { + params["freshness_strict"] = c.freshnessStrict + } +} + +func NoneConsistency() Consistency { + return Consistency{ + level: ConsistencyLevelNone, + } +} + +func WeakConsistency() Consistency { + return Consistency{ + level: ConsistencyLevelWeak, + } +} + +func StrongConsistency() Consistency { + return Consistency{ + level: ConsistencyLevelStrong, + } +} diff --git a/service/rqlite/db/db.go b/service/rqlite/db/db.go index 9367e85..2d7a72a 100644 --- a/service/rqlite/db/db.go +++ b/service/rqlite/db/db.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" + "github.com/iancoleman/strcase" "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/tidwall/gjson" @@ -11,6 +12,10 @@ import ( "github.com/txix-open/isp-kit/json" ) +func init() { + sqlx.NameMapper = strcase.ToSnake +} + type DB struct { cli *httpcli.Client } @@ -37,11 +42,14 @@ func (d DB) Select(ctx context.Context, ptr any, query string, args ...any) erro resp := Response{ Results: []*Result{result}, } + params := map[string]any{ + "timings": true, + "associative": true, + } + ConsistencyFromContext(ctx).appendParams(params) err := d.cli.Post("/db/query"). - QueryParams(map[string]any{ - "timings": true, - "associative": true, - }).JsonRequestBody(Request(query, args...)). + QueryParams(params). + JsonRequestBody(request(query, args...)). JsonResponseBody(&resp). StatusCodeToError(). DoWithoutResponse(ctx) @@ -59,11 +67,14 @@ func (d DB) SelectRow(ctx context.Context, ptr any, query string, args ...any) e resp := Response{ Results: []*Result{result}, } - httpResp, err := d.cli.Post("/db/query"). - QueryParams(map[string]any{ - "timings": true, - "associative": true, - }).JsonRequestBody(Request(query, args...)). + params := map[string]any{ + "timings": true, + "associative": true, + } + ConsistencyFromContext(ctx).appendParams(params) + httpResp, err := d.cli.Post("/db/request"). + QueryParams(params). + JsonRequestBody(requests(request(query, args...))). JsonResponseBody(&resp). StatusCodeToError(). Do(ctx) @@ -98,7 +109,7 @@ func (d DB) Exec(ctx context.Context, query string, args ...any) (sql.Result, er err := d.cli.Post("/db/execute"). QueryParams(map[string]any{ "timings": true, - }).JsonRequestBody(Request(query, args...)). + }).JsonRequestBody(requests(request(query, args...))). JsonResponseBody(&resp). StatusCodeToError(). DoWithoutResponse(ctx) diff --git a/service/rqlite/db/request.go b/service/rqlite/db/request.go index 75a4afa..789868c 100644 --- a/service/rqlite/db/request.go +++ b/service/rqlite/db/request.go @@ -1,5 +1,9 @@ package db -func Request(query string, args ...any) []any { +func request(query string, args ...any) []any { return append([]any{query}, args...) } + +func requests(requests ...[]any) [][]any { + return requests +} diff --git a/service/startup/service.go b/service/startup/service.go index e673540..386bf4c 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -13,6 +13,7 @@ import ( "github.com/txix-open/isp-kit/dbx/migration" "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/http" + "github.com/txix-open/isp-kit/http/httpcli" "github.com/txix-open/isp-kit/http/httpclix" "github.com/txix-open/isp-kit/log" "isp-config-service/assembly" @@ -80,7 +81,7 @@ func (s Service) Run(ctx context.Context) error { s.logger.Debug(ctx, "is not a leader") } - db, err := db.Open(ctx, s.rqlite.Dsn(), httpclix.Default()) + db, err := db.Open(ctx, s.rqlite.Dsn(), httpclix.Default(httpcli.WithMiddlewares(httpclix.Log(s.logger)))) if err != nil { return errors.WithMessage(err, "dial to embedded rqlite") } From c4abdaa54b63449edfc4ac4ea298e2905d28fc6e Mon Sep 17 00:00:00 2001 From: d1slike Date: Mon, 6 May 2024 20:09:14 +0300 Subject: [PATCH 04/40] Implement lifecycle --- assembly/locator.go | 21 +++- controller/module.go | 30 +++--- entity/backend.go | 17 ++-- entity/config.go | 47 +++++---- entity/event.go | 47 +++++++++ entity/json.go | 45 --------- entity/module.go | 12 +-- entity/xtypes/bool.go | 25 +++++ entity/xtypes/json.go | 39 ++++++++ entity/xtypes/time.go | 32 ++++++ helpers/helpers.go | 10 ++ migrations/20240503120009_init.sql | 74 +++++++------- repository/backend.go | 2 +- repository/config.go | 64 ++++++++++++ repository/config_schema.go | 39 ++++++++ repository/event.go | 54 ++++++++++ repository/module.go | 7 +- service/module.go | 152 ++++++++++++++++++++++++----- service/rqlite/db/consistency.go | 2 +- service/rqlite/db/db.go | 4 +- service/startup/service.go | 32 +++--- 21 files changed, 582 insertions(+), 173 deletions(-) create mode 100644 entity/event.go delete mode 100644 entity/json.go create mode 100644 entity/xtypes/bool.go create mode 100644 entity/xtypes/json.go create mode 100644 entity/xtypes/time.go create mode 100644 helpers/helpers.go create mode 100644 repository/config.go create mode 100644 repository/config_schema.go create mode 100644 repository/event.go diff --git a/assembly/locator.go b/assembly/locator.go index 325304b..c2abdbe 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -8,7 +8,10 @@ import ( "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/grpc/endpoint" "github.com/txix-open/isp-kit/log" + "isp-config-service/controller" + "isp-config-service/repository" "isp-config-service/routes" + "isp-config-service/service" ) const ( @@ -34,7 +37,23 @@ type Config struct { } func (l Locator) Config() Config { - controllers := routes.Controllers{} + moduleRepo := repository.NewModule(l.db) + backendRepo := repository.NewBackend(l.db) + eventRepo := repository.NewEvent(l.db) + configRepo := repository.NewConfig(l.db) + configSchemaRepo := repository.NewConfigSchema(l.db) + moduleService := service.NewModule( + moduleRepo, + backendRepo, + eventRepo, + configRepo, + configSchemaRepo, + l.logger, + ) + moduleController := controller.NewModule(moduleService, l.logger) + controllers := routes.Controllers{ + Module: moduleController, + } mapper := endpoint.DefaultWrapper(l.logger) grpcMux := routes.GrpcHandler(mapper, controllers) diff --git a/controller/module.go b/controller/module.go index 1a1d7d0..ee2b9ab 100644 --- a/controller/module.go +++ b/controller/module.go @@ -9,6 +9,7 @@ import ( "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/json" "github.com/txix-open/isp-kit/log" + "isp-config-service/helpers" ) var ( @@ -16,9 +17,9 @@ var ( ) type ModuleService interface { - OnConnect(conn *etp.Conn, moduleName string) error - OnDisconnect(conn *etp.Conn, moduleName string, isNormalClose bool, err error) error - OnError(conn *etp.Conn, moduleName string, err error) + OnConnect(ctx context.Context, conn *etp.Conn, moduleName string) error + OnDisconnect(ctx context.Context, conn *etp.Conn, moduleName string, isNormalClose bool, err error) error + OnError(ctx context.Context, conn *etp.Conn, moduleName string, err error) OnModuleReady( ctx context.Context, conn *etp.Conn, @@ -49,26 +50,29 @@ func NewModule(service ModuleService, logger log.Logger) Module { } func (m Module) OnConnect(conn *etp.Conn) { - err := m.service.OnConnect(conn, moduleName(conn)) + ctx := conn.HttpRequest().Context() + err := m.service.OnConnect(ctx, conn, helpers.ModuleName(conn)) if err != nil { - m.handleError(context.Background(), err) + m.handleError(ctx, errors.WithMessage(err, "handle onConnect")) } } func (m Module) OnDisconnect(conn *etp.Conn, err error) { + ctx := conn.HttpRequest().Context() handleDisconnectErr := m.service.OnDisconnect( + ctx, conn, - moduleName(conn), + helpers.ModuleName(conn), etp.IsNormalClose(err), err, ) if handleDisconnectErr != nil { - m.handleError(context.Background(), handleDisconnectErr) + m.handleError(ctx, errors.WithMessage(handleDisconnectErr, "handle onDisconnect")) } } func (m Module) OnError(conn *etp.Conn, err error) { - m.service.OnError(conn, moduleName(conn), err) + m.service.OnError(conn.HttpRequest().Context(), conn, helpers.ModuleName(conn), err) } func (m Module) OnModuleReady(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { @@ -80,7 +84,7 @@ func (m Module) OnModuleReady(ctx context.Context, conn *etp.Conn, event msg.Eve err = m.service.OnModuleReady(ctx, conn, backend) if err != nil { - return m.handleError(ctx, err) + return m.handleError(ctx, errors.WithMessage(err, "handle onModuleReady")) } return ok @@ -95,7 +99,7 @@ func (m Module) OnModuleRequirements(ctx context.Context, conn *etp.Conn, event err = m.service.OnModuleRequirements(ctx, conn, requirements) if err != nil { - return m.handleError(ctx, err) + return m.handleError(ctx, errors.WithMessage(err, "handle onModuleRequirements")) } return ok @@ -110,7 +114,7 @@ func (m Module) OnModuleConfigSchema(ctx context.Context, conn *etp.Conn, event err = m.service.OnModuleConfigSchema(ctx, conn, configData) if err != nil { - return m.handleError(ctx, err) + return m.handleError(ctx, errors.WithMessage(err, "handle onModuleConfigSchema")) } return ok @@ -123,7 +127,3 @@ func (m Module) handleError( m.logger.Error(ctx, err) return []byte(err.Error()) } - -func moduleName(conn *etp.Conn) string { - return conn.HttpRequest().Form.Get("module_name") -} diff --git a/entity/backend.go b/entity/backend.go index e7df19a..51d1d38 100644 --- a/entity/backend.go +++ b/entity/backend.go @@ -2,15 +2,16 @@ package entity import ( "github.com/txix-open/isp-kit/cluster" + "isp-config-service/entity/xtypes" ) type Backend struct { - ModuleId string `json:"module_id"` - Address string `json:"address"` - Version string `json:"version"` - LibVersion string `json:"lib_version"` - Endpoints JsonValue[[]cluster.EndpointDescriptor] `json:"endpoints"` - RequiredModules JsonValue[cluster.ModuleRequirements] `json:"required_modules"` - //CreatedAt time.Time `json:"created_at"` - //UpdatedAt time.Time `json:"updated_at"` + ModuleId string `json:"module_id"` + Address string `json:"address"` + Version string `json:"version"` + LibVersion string `json:"lib_version"` + Endpoints xtypes.Json[[]cluster.EndpointDescriptor] `json:"endpoints"` + RequiredModules xtypes.Json[[]cluster.ModuleDependency] `json:"required_modules"` + CreatedAt xtypes.Time `json:"created_at"` + UpdatedAt xtypes.Time `json:"updated_at"` } diff --git a/entity/config.go b/entity/config.go index fd36826..46313d5 100644 --- a/entity/config.go +++ b/entity/config.go @@ -1,34 +1,39 @@ package entity import ( - "time" + "github.com/pkg/errors" + "isp-config-service/entity/xtypes" +) + +var ( + ErrNoActiveConfig = errors.New("no active config") ) type Config struct { - Id string `json:"id"` - Name string `json:"name"` - ModuleId string `json:"module_id"` - Data []byte `json:"data"` - Version int `json:"version"` - Active int `json:"active"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + Id string `json:"id"` + Name string `json:"name"` + ModuleId string `json:"module_id"` + Data []byte `json:"data"` + Version int `json:"version"` + Active xtypes.Bool `json:"active"` + CreatedAt xtypes.Time `json:"created_at"` + UpdatedAt xtypes.Time `json:"updated_at"` } type ConfigHistory struct { - Id string `json:"id"` - ConfigId string `json:"config_id"` - Data []byte `json:"data"` - Version int `json:"version"` - AdminId int `json:"admin_id"` - CreatedAt time.Time `json:"created_at"` + Id string `json:"id"` + ConfigId string `json:"config_id"` + Data []byte `json:"data"` + Version int `json:"version"` + AdminId int `json:"admin_id"` + CreatedAt xtypes.Time `json:"created_at"` } type ConfigSchema struct { - Id string `json:"id"` - ModuleId string `json:"module_id"` - Data []byte `json:"data"` - ModuleVersion string `json:"module_version"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + Id string `json:"id"` + ModuleId string `json:"module_id"` + Data []byte `json:"data"` + ModuleVersion string `json:"module_version"` + CreatedAt xtypes.Time `json:"created_at"` + UpdatedAt xtypes.Time `json:"updated_at"` } diff --git a/entity/event.go b/entity/event.go new file mode 100644 index 0000000..44e006b --- /dev/null +++ b/entity/event.go @@ -0,0 +1,47 @@ +package entity + +import ( + "isp-config-service/entity/xtypes" +) + +type EventType int + +const ( + Unknown EventType = iota + ConfigUpdated + ModuleReady + ModuleDisconnected +) + +type Event struct { + RowId int `json:"rowid"` + Type EventType `json:"type"` + Payload xtypes.Json[EventPayload] `json:"payload"` +} + +func NewEvent(t EventType, payload EventPayload) Event { + return Event{ + Type: t, + Payload: xtypes.Json[EventPayload]{ + Value: payload, + }, + } +} + +type EventPayload struct { + ConfigUpdated *PayloadConfigUpdated `json:",omitempty"` + ModuleReady *PayloadModuleReady `json:",omitempty"` + ModuleDisconnected *PayloadModuleDisconnected `json:",omitempty"` +} + +type PayloadConfigUpdated struct { + ModuleId string +} + +type PayloadModuleReady struct { + ModuleId string +} + +type PayloadModuleDisconnected struct { + ModuleId string +} diff --git a/entity/json.go b/entity/json.go deleted file mode 100644 index 4165bdb..0000000 --- a/entity/json.go +++ /dev/null @@ -1,45 +0,0 @@ -package entity - -import ( - "encoding/base64" - "strconv" - - "github.com/txix-open/isp-kit/json" -) - -type JsonValue[T any] struct { - Value T -} - -func (l JsonValue[T]) MarshalJSON() ([]byte, error) { - data, err := l.MarshalText() - if err != nil { - return nil, err - } - return []byte(strconv.Quote(string(data))), nil -} - -func (l *JsonValue[T]) UnmarshalJSON(data []byte) error { - s, err := strconv.Unquote(string(data)) - if err != nil { - return err - } - return l.UnmarshalText([]byte(s)) -} - -func (l JsonValue[T]) MarshalText() ([]byte, error) { - data, err := json.Marshal(l.Value) - if err != nil { - return nil, err - } - s := base64.StdEncoding.EncodeToString(data) - return []byte(s), nil -} - -func (l *JsonValue[T]) UnmarshalText(data []byte) error { - arr, err := base64.StdEncoding.DecodeString(string(data)) - if err != nil { - return err - } - return json.Unmarshal(arr, &l.Value) -} diff --git a/entity/module.go b/entity/module.go index 5d1836f..ccfe358 100644 --- a/entity/module.go +++ b/entity/module.go @@ -1,13 +1,13 @@ package entity import ( - "time" + "isp-config-service/entity/xtypes" ) type Module struct { - Id string `json:"id"` - Name string `json:"name"` - LastConnectedAt *time.Time `json:"last_connected_at"` - LastDisconnectedAt *time.Time `json:"last_disconnected_at"` - CreatedAt time.Time `json:"created_at"` + Id string `json:"id"` + Name string `json:"name"` + LastConnectedAt *xtypes.Time `json:"last_connected_at"` + LastDisconnectedAt *xtypes.Time `json:"last_disconnected_at"` + CreatedAt xtypes.Time `json:"created_at"` } diff --git a/entity/xtypes/bool.go b/entity/xtypes/bool.go new file mode 100644 index 0000000..839cf3e --- /dev/null +++ b/entity/xtypes/bool.go @@ -0,0 +1,25 @@ +package xtypes + +type Bool struct { + Value bool +} + +func (l Bool) MarshalJSON() ([]byte, error) { + return l.MarshalText() +} + +func (l *Bool) UnmarshalJSON(data []byte) error { + return l.UnmarshalText(data) +} + +func (l Bool) MarshalText() ([]byte, error) { + if l.Value { + return []byte("1"), nil + } + return []byte("0"), nil +} + +func (l *Bool) UnmarshalText(data []byte) error { + l.Value = string(data) == "1" + return nil +} diff --git a/entity/xtypes/json.go b/entity/xtypes/json.go new file mode 100644 index 0000000..8d90ccb --- /dev/null +++ b/entity/xtypes/json.go @@ -0,0 +1,39 @@ +package xtypes + +import ( + "strconv" + + "github.com/txix-open/isp-kit/json" +) + +type Json[T any] struct { + Value T +} + +func (l Json[T]) MarshalJSON() ([]byte, error) { + data, err := l.MarshalText() + if err != nil { + return nil, err + } + return []byte(strconv.Quote(string(data))), nil +} + +func (l *Json[T]) UnmarshalJSON(data []byte) error { + s, err := strconv.Unquote(string(data)) + if err != nil { + return err + } + return l.UnmarshalText([]byte(s)) +} + +func (l Json[T]) MarshalText() ([]byte, error) { + data, err := json.Marshal(l.Value) + if err != nil { + return nil, err + } + return data, nil +} + +func (l *Json[T]) UnmarshalText(data []byte) error { + return json.Unmarshal(data, &l.Value) +} diff --git a/entity/xtypes/time.go b/entity/xtypes/time.go new file mode 100644 index 0000000..316224c --- /dev/null +++ b/entity/xtypes/time.go @@ -0,0 +1,32 @@ +package xtypes + +import ( + "strconv" + "time" +) + +type Time struct { + Value time.Time +} + +func (l Time) MarshalJSON() ([]byte, error) { + return l.MarshalText() +} + +func (l *Time) UnmarshalJSON(data []byte) error { + return l.UnmarshalText(data) +} + +func (l Time) MarshalText() ([]byte, error) { + value := strconv.Itoa(int(l.Value.Unix())) + return []byte(value), nil +} + +func (l *Time) UnmarshalText(data []byte) error { + value, err := strconv.Atoi(string(data)) + if err != nil { + return err + } + l.Value = time.Unix(int64(value), 0) + return nil +} diff --git a/helpers/helpers.go b/helpers/helpers.go new file mode 100644 index 0000000..ef77860 --- /dev/null +++ b/helpers/helpers.go @@ -0,0 +1,10 @@ +package helpers + +import ( + "github.com/txix-open/etp/v3" +) + +func ModuleName(conn *etp.Conn) string { + _ = conn.HttpRequest().ParseForm() + return conn.HttpRequest().Form.Get("module_name") +} diff --git a/migrations/20240503120009_init.sql b/migrations/20240503120009_init.sql index 1fbddd1..fb2555d 100644 --- a/migrations/20240503120009_init.sql +++ b/migrations/20240503120009_init.sql @@ -2,23 +2,23 @@ -- +goose NO TRANSACTION create table isp_config_service__module ( - id text not null primary key, - name text not null unique, - last_connected_at text, - last_disconnected_at text, - created_at text not null default (datetime('now')) + id text not null primary key, + name text not null unique, + last_connected_at integer, + last_disconnected_at integer, + created_at integer not null default (unixepoch()) ); create table isp_config_service__config ( - id text not null primary key, - name text not null, - module_id text not null, - data blob not null, - version int not null default 1, - active int not null default 0, - created_at text not null default (datetime('now')), - updated_at text not null default (datetime('now')), + id text not null primary key, + name text not null, + module_id text not null, + data blob not null, + version int not null default 1, + active int not null default 0, + created_at integer not null default (unixepoch()), + updated_at integer not null default (unixepoch()), foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade ); @@ -26,12 +26,12 @@ create index IX_isp_config_service__config__module_id on isp_config_service__con create table isp_config_service__config_history ( - id text not null primary key, - config_id text not null, - data blob not null, - version int not null default 1, - admin_id int not null, - created_at text not null default (datetime('now')), + id text not null primary key, + config_id text not null, + data blob not null, + version int not null default 1, + admin_id int not null, + created_at integer not null default (unixepoch()), foreign key (config_id) references isp_config_service__config (id) on delete cascade on update cascade ); @@ -39,32 +39,40 @@ create index IX_isp_config_service__config_history__config_id on isp_config_serv create table isp_config_service__config_schema ( - id text not null primary key, - module_id text not null unique, - data blob not null, - module_version text not null default 1, - created_at text not null default (datetime('now')), - updated_at text not null default (datetime('now')), + id text not null primary key, + module_id text not null unique, + data blob not null, + module_version text not null default 1, + created_at integer not null default (unixepoch()), + updated_at integer not null default (unixepoch()), foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade ); create table isp_config_service__backend ( - module_id text not null, - address text not null, - version text not null, - lib_version text not null, - endpoints blob not null, - required_modules blob not null, - created_at text not null default (datetime('now')), - updated_at text not null default (datetime('now')), + module_id text not null, + address text not null, + version text not null, + lib_version text not null, + endpoints blob not null, + required_modules blob not null, + created_at integer not null default (unixepoch()), + updated_at integer not null default (unixepoch()), primary key (module_id, address), foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade ); +create table isp_config_service__event +( + type integer not null, + payload blob not null, + created_at integer not null default (unixepoch()) +); + -- +goose Down drop table isp_config_service__module; drop table isp_config_service__config; drop table isp_config_service__config_history; drop table isp_config_service__config_schema; drop table isp_config_service__backend; +drop table isp_config_service__event; diff --git a/repository/backend.go b/repository/backend.go index c589f59..b8a229c 100644 --- a/repository/backend.go +++ b/repository/backend.go @@ -30,7 +30,7 @@ func (r Backend) Upsert(ctx context.Context, backend entity.Backend) error { ).Suffix(`on conflict (module_id, address) do update set version = excluded.version, lib_version = excluded.lib_version, endpoints = excluded.endpoints, required_modules = excluded.required_modules, - updated_at = datetime('now') + updated_at = unixepoch() `). ToSql() if err != nil { diff --git a/repository/config.go b/repository/config.go new file mode 100644 index 0000000..fa198df --- /dev/null +++ b/repository/config.go @@ -0,0 +1,64 @@ +package repository + +import ( + "context" + "database/sql" + + "github.com/Masterminds/squirrel" + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/db" + "isp-config-service/entity" +) + +type Config struct { + db db.DB +} + +func NewConfig(db db.DB) Config { + return Config{ + db: db, + } +} + +func (r Config) Insert(ctx context.Context, cfg entity.Config) error { + query, args, err := squirrel.Insert(Table("config")). + Columns("id", "name", "module_id", "data", "version", "active"). + Values(cfg.Id, cfg.Name, cfg.ModuleId, cfg.Data, cfg.Version, cfg.Active). + Suffix("on conflict (id) do nothing"). + ToSql() + if err != nil { + return errors.WithMessage(err, "build query") + } + + _, err = r.db.Exec(ctx, query, args...) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + + return nil +} + +func (r Config) GetActive(ctx context.Context, moduleId string) (*entity.Config, error) { + query, args, err := squirrel.Select("*"). + From(Table("config")). + Where(squirrel.Eq{ + "module_id": moduleId, + "active": "1", + }).OrderBy("version DESC"). + Limit(1). + ToSql() + if err != nil { + return nil, errors.WithMessage(err, "build query") + } + + result := entity.Config{} + err = r.db.SelectRow(ctx, &result, query, args...) + if errors.Is(err, sql.ErrNoRows) { + return nil, entity.ErrNoActiveConfig + } + if err != nil { + return nil, errors.WithMessagef(err, "select row: %s", query) + } + + return &result, nil +} diff --git a/repository/config_schema.go b/repository/config_schema.go new file mode 100644 index 0000000..46e9204 --- /dev/null +++ b/repository/config_schema.go @@ -0,0 +1,39 @@ +package repository + +import ( + "context" + + "github.com/Masterminds/squirrel" + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/db" + "isp-config-service/entity" +) + +type ConfigSchema struct { + db db.DB +} + +func NewConfigSchema(db db.DB) ConfigSchema { + return ConfigSchema{ + db: db, + } +} + +func (r ConfigSchema) Upsert(ctx context.Context, schema entity.ConfigSchema) error { + query, args, err := squirrel.Insert(Table("config_schema")). + Columns("id", "module_id", "data", "module_version"). + Values(schema.Id, schema.ModuleId, schema.Data, schema.ModuleVersion). + Suffix(`on conflict (module_id) do update + set data = excluded.data, module_version = excluded.module_version, updated_at = unixepoch()`). + ToSql() + if err != nil { + return errors.WithMessage(err, "build query") + } + + _, err = r.db.Exec(ctx, query, args...) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + + return nil +} diff --git a/repository/event.go b/repository/event.go new file mode 100644 index 0000000..3e15aa3 --- /dev/null +++ b/repository/event.go @@ -0,0 +1,54 @@ +package repository + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/db" + "isp-config-service/entity" +) + +type Event struct { + db db.DB +} + +func NewEvent(db db.DB) Event { + return Event{ + db: db, + } +} + +func (r Event) Insert(ctx context.Context, event entity.Event) error { + query := fmt.Sprintf("insert into %s (type, payload) values (?, ?)", Table("event")) + _, err := r.db.Exec(ctx, query, event.Type, event.Payload) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + return nil +} + +func (r Event) Get(ctx context.Context, lastRowId int, limit int) ([]entity.Event, error) { + query := fmt.Sprintf("select * from %s where rowid > ? order by rowid desc limit ?", Table("event")) + result := make([]entity.Event, 0) + err := r.db.Select(ctx, &result, query, lastRowId, limit) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + return result, nil +} + +func (r Event) DeleteByCreatedAt(ctx context.Context, before int) (int, error) { + query := fmt.Sprintf("delete from %s where created_at < ?", Table("event")) + result, err := r.db.Exec(ctx, query, before) + if err != nil { + return 0, errors.WithMessagef(err, "exec: %s", query) + } + + affected, err := result.RowsAffected() + if err != nil { + return 0, errors.WithMessage(err, "get affected rows") + } + + return int(affected), nil +} diff --git a/repository/module.go b/repository/module.go index 462054f..f7c9184 100644 --- a/repository/module.go +++ b/repository/module.go @@ -3,12 +3,12 @@ package repository import ( "context" "fmt" - "time" "github.com/Masterminds/squirrel" "github.com/pkg/errors" "github.com/txix-open/isp-kit/db" "isp-config-service/entity" + "isp-config-service/entity/xtypes" ) type Module struct { @@ -25,7 +25,8 @@ func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error query, args, err := squirrel.Insert(Table("module")). Columns("id", "name", "last_connected_at"). Values(module.Id, module.Name, module.LastConnectedAt). - Suffix("on conflict (name) do update set last_connected_at = excluded.last_connected_at"). + Suffix(`on conflict (name) do update + set last_connected_at = excluded.last_connected_at`). Suffix("returning id"). ToSql() if err != nil { @@ -43,7 +44,7 @@ func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error func (r Module) SetDisconnectedAt( ctx context.Context, moduleId string, - disconnected time.Time, + disconnected xtypes.Time, ) error { query, args, err := squirrel.Update(Table("module")). Set("last_disconnected_at", disconnected). diff --git a/service/module.go b/service/module.go index 8fe9c3f..a00d85e 100644 --- a/service/module.go +++ b/service/module.go @@ -2,6 +2,8 @@ package service import ( "context" + "crypto/md5" + "encoding/hex" "fmt" "time" @@ -12,6 +14,8 @@ import ( "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/log" "isp-config-service/entity" + "isp-config-service/entity/xtypes" + "isp-config-service/helpers" ) const ( @@ -24,7 +28,7 @@ type ModuleRepo interface { SetDisconnectedAt( ctx context.Context, moduleId string, - disconnected time.Time, + disconnected xtypes.Time, ) error } @@ -33,33 +37,54 @@ type BackendRepo interface { Delete(ctx context.Context, moduleId string, address string) error } +type EventRepo interface { + Insert(ctx context.Context, event entity.Event) error +} + +type ConfigSchemaRepo interface { + Upsert(ctx context.Context, schema entity.ConfigSchema) error +} + +type ConfigRepo interface { + Insert(ctx context.Context, cfg entity.Config) error + GetActive(ctx context.Context, moduleId string) (*entity.Config, error) +} + type Module struct { - moduleRepo ModuleRepo - backendRepo BackendRepo - logger log.Logger + moduleRepo ModuleRepo + backendRepo BackendRepo + eventRepo EventRepo + configRepo ConfigRepo + configSchemaRepo ConfigSchemaRepo + logger log.Logger } func NewModule( moduleRepo ModuleRepo, backendRepo BackendRepo, + eventRepo EventRepo, + configRepo ConfigRepo, + configSchemaRepo ConfigSchemaRepo, logger log.Logger, ) Module { return Module{ - moduleRepo: moduleRepo, - backendRepo: backendRepo, - logger: logger, + moduleRepo: moduleRepo, + backendRepo: backendRepo, + eventRepo: eventRepo, + configRepo: configRepo, + configSchemaRepo: configSchemaRepo, + logger: logger, } } -func (s Module) OnConnect(conn *etp.Conn, moduleName string) error { - now := time.Now().UTC() +func (s Module) OnConnect(ctx context.Context, conn *etp.Conn, moduleName string) error { + now := now() module := entity.Module{ Id: uuid.NewString(), Name: moduleName, - LastConnectedAt: &now, - CreatedAt: now, + LastConnectedAt: &xtypes.Time{Value: now}, } - moduleId, err := s.moduleRepo.Upsert(context.Background(), module) + moduleId, err := s.moduleRepo.Upsert(ctx, module) if err != nil { return errors.WithMessage(err, "upsert module in store") } @@ -70,25 +95,27 @@ func (s Module) OnConnect(conn *etp.Conn, moduleName string) error { } func (s Module) OnDisconnect( + ctx context.Context, conn *etp.Conn, moduleName string, isNormalClose bool, err error, ) error { if isNormalClose { - s.logger.Info(context.Background(), fmt.Sprintf("module '%s' disconnected", moduleName)) + s.logger.Info(ctx, fmt.Sprintf("module '%s' disconnected", moduleName)) } else { message := errors.WithMessagef( err, "module '%s' unexpectedly disconnected", moduleName, ) - s.logger.Error(context.Background(), message) + s.logger.Error(ctx, message) } + now := now() moduleId, _ := store.Get[string](conn.Data(), moduleIdKey) if moduleId != "" { - err = s.moduleRepo.SetDisconnectedAt(context.Background(), moduleName, time.Now().UTC()) + err = s.moduleRepo.SetDisconnectedAt(ctx, moduleName, xtypes.Time{Value: now}) if err != nil { return errors.WithMessage(err, "update disconnected time in store") } @@ -96,28 +123,38 @@ func (s Module) OnDisconnect( backend, _ := store.Get[entity.Backend](conn.Data(), backendKey) if backend.ModuleId != "" { - err = s.backendRepo.Delete(context.Background(), backend.ModuleId, backend.Address) + err = s.backendRepo.Delete(ctx, backend.ModuleId, backend.Address) if err != nil { return errors.WithMessage(err, "delete backend in store") } + + event := entity.NewEvent(entity.ModuleDisconnected, entity.EventPayload{ + ModuleDisconnected: &entity.PayloadModuleDisconnected{ + ModuleId: moduleId, + }, + }) + err = s.eventRepo.Insert(ctx, event) + if err != nil { + return errors.WithMessage(err, "insert event in store") + } } return nil } -func (s Module) OnError(conn *etp.Conn, moduleName string, err error) { +func (s Module) OnError(ctx context.Context, conn *etp.Conn, moduleName string, err error) { err = errors.WithMessagef( err, "unexpected error in communication, module: '%s'", moduleName, ) - s.logger.Error(context.Background(), err) + s.logger.Error(ctx, err) } func (s Module) OnModuleReady( ctx context.Context, conn *etp.Conn, - backend cluster.BackendDeclaration, + declaration cluster.BackendDeclaration, ) error { moduleId, err := store.Get[string](conn.Data(), moduleIdKey) if err != nil { @@ -126,14 +163,30 @@ func (s Module) OnModuleReady( backend := entity.Backend{ ModuleId: moduleId, - Address: fmt.Sprintf("%s:%s", backend.Address.IP, backend.Address.Port), - Version: backend.Version, - LibVersion: backend.LibVersion, - Endpoints: nil, - RequiredModules: nil, - CreatedAt: time.Time{}, - UpdatedAt: time.Time{}, + Address: fmt.Sprintf("%s:%s", declaration.Address.IP, declaration.Address.Port), + Version: declaration.Version, + LibVersion: declaration.LibVersion, + Endpoints: xtypes.Json[[]cluster.EndpointDescriptor]{Value: declaration.Endpoints}, + RequiredModules: xtypes.Json[[]cluster.ModuleDependency]{Value: declaration.RequiredModules}, } + err = s.backendRepo.Upsert(ctx, backend) + if err != nil { + return errors.WithMessage(err, "upsert backend in store") + } + + conn.Data().Set(backendKey, backend) + + event := entity.NewEvent(entity.ModuleReady, entity.EventPayload{ + ModuleReady: &entity.PayloadModuleReady{ + ModuleId: moduleId, + }, + }) + err = s.eventRepo.Insert(ctx, event) + if err != nil { + return errors.WithMessage(err, "insert event in store") + } + + return nil } func (s Module) OnModuleRequirements( @@ -149,5 +202,52 @@ func (s Module) OnModuleConfigSchema( conn *etp.Conn, data cluster.ConfigData, ) error { + moduleId, err := store.Get[string](conn.Data(), moduleIdKey) + if err != nil { + return errors.WithMessage(err, "resolve module id") + } + + config, err := s.configRepo.GetActive(ctx, moduleId) + if errors.Is(err, entity.ErrNoActiveConfig) { + md5Sum := md5.Sum([]byte(moduleId)) + initialConfigId := hex.EncodeToString(md5Sum[:]) + initialConfig := entity.Config{ + Id: initialConfigId, + Name: helpers.ModuleName(conn), + ModuleId: moduleId, + Data: data.Config, + Version: 1, + Active: xtypes.Bool{Value: true}, + } + err := s.configRepo.Insert(ctx, initialConfig) + if err != nil { + return errors.WithMessage(err, "insert config in store") + } + config = &initialConfig + } + if err != nil { + return errors.WithMessage(err, "get active config") + } + + err = conn.Emit(ctx, cluster.ConfigSendConfigWhenConnected, config.Data) + if err != nil { + return errors.WithMessage(err, "send event with config") + } + + configSchema := entity.ConfigSchema{ + Id: uuid.NewString(), + ModuleId: moduleId, + Data: data.Schema, + ModuleVersion: data.Version, + } + err = s.configSchemaRepo.Upsert(ctx, configSchema) + if err != nil { + return errors.WithMessage(err, "upsert config schema in store") + } + return nil } + +func now() time.Time { + return time.Now().UTC() +} diff --git a/service/rqlite/db/consistency.go b/service/rqlite/db/consistency.go index 510373b..673704a 100644 --- a/service/rqlite/db/consistency.go +++ b/service/rqlite/db/consistency.go @@ -11,7 +11,7 @@ var ( consistencyCtxValue = consistencyCtxKey{} ) -func ConsistencyFromContext(ctx context.Context) Consistency { +func consistencyFromContext(ctx context.Context) Consistency { c, ok := ctx.Value(consistencyCtxValue).(Consistency) if !ok { return Consistency{} diff --git a/service/rqlite/db/db.go b/service/rqlite/db/db.go index 2d7a72a..8c3a912 100644 --- a/service/rqlite/db/db.go +++ b/service/rqlite/db/db.go @@ -46,7 +46,7 @@ func (d DB) Select(ctx context.Context, ptr any, query string, args ...any) erro "timings": true, "associative": true, } - ConsistencyFromContext(ctx).appendParams(params) + consistencyFromContext(ctx).appendParams(params) err := d.cli.Post("/db/query"). QueryParams(params). JsonRequestBody(request(query, args...)). @@ -71,7 +71,7 @@ func (d DB) SelectRow(ctx context.Context, ptr any, query string, args ...any) e "timings": true, "associative": true, } - ConsistencyFromContext(ctx).appendParams(params) + consistencyFromContext(ctx).appendParams(params) httpResp, err := d.cli.Post("/db/request"). QueryParams(params). JsonRequestBody(requests(request(query, args...))). diff --git a/service/startup/service.go b/service/startup/service.go index 386bf4c..f47e5bd 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -10,6 +10,7 @@ import ( "github.com/txix-open/etp/v3" "github.com/txix-open/isp-kit/app" "github.com/txix-open/isp-kit/bootstrap" + "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/dbx/migration" "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/http" @@ -28,11 +29,12 @@ const ( ) type Service struct { - boot *bootstrap.Bootstrap - rqlite *rqlite.Rqlite - grpcSrv *grpc.Server - httpSrv *http.Server - logger log.Logger + boot *bootstrap.Bootstrap + rqlite *rqlite.Rqlite + grpcSrv *grpc.Server + httpSrv *http.Server + clusterCli *cluster.Client + logger log.Logger //initialized in Run etpSrv *etp.Server @@ -41,11 +43,12 @@ type Service struct { func New(boot *bootstrap.Bootstrap) Service { rqlite := rqlite.New(boot.App.Config()) return Service{ - boot: boot, - rqlite: rqlite, - logger: boot.App.Logger(), - httpSrv: http.NewServer(boot.App.Logger()), - grpcSrv: grpc.DefaultServer(), + boot: boot, + rqlite: rqlite, + grpcSrv: grpc.DefaultServer(), + httpSrv: http.NewServer(boot.App.Logger()), + clusterCli: boot.ClusterCli, + logger: boot.App.Logger(), } } @@ -107,7 +110,14 @@ func (s Service) Run(ctx context.Context) error { s.boot.Fatal(errors.WithMessage(err, "start http server")) } }() - time.Sleep(1 * time.Second) + time.Sleep(1 * time.Second) //wait for http start + + go func() { + err = s.clusterCli.Run(ctx, cluster.NewEventHandler()) + if err != nil { + s.boot.Fatal(errors.WithMessage(err, "connect to it's self")) + } + }() return nil } From f90063cb03dbe43ad06d551e1dbbf5e45a63c226 Mon Sep 17 00:00:00 2001 From: d1slike Date: Mon, 6 May 2024 20:18:15 +0300 Subject: [PATCH 05/40] Add go.sum --- go.sum | 375 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 go.sum diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..18095f5 --- /dev/null +++ b/go.sum @@ -0,0 +1,375 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= +github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA= +github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-metrics v0.5.1 h1:rfPwUqFU6uZXNvGl4hzjY8LEBsqFVU4si1H9/Hqck/U= +github.com/hashicorp/go-metrics v0.5.1/go.mod h1:KEjodfebIOuBYSAe/bHTm+HChmKSxAOXPBieMLYozDE= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= +github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= +github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0= +github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft v1.6.1 h1:v/jm5fcYHvVkL0akByAp+IDdDSzCNCGhdO6VdB56HIM= +github.com/hashicorp/raft v1.6.1/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0= +github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0= +github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= +github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= +github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= +github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rqlite/go-sqlite3 v1.32.0 h1:bB0IUDX8bswSpAmxltWtu/la2DV/GAccjprOcUEdihY= +github.com/rqlite/go-sqlite3 v1.32.0/go.mod h1:R9H7CatgYBt3c+fSV/5yo2vLh4ZjCB0aMHdkv69fP4A= +github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 h1:gYUQqzapdN4PQF5j0zDFI9ANQVAVFoJivNp5bTZEZMo= +github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= +github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 h1:NZ62M+kT0JqhyFUMc8I4SMmfmD4NGJxhb2ePJQXjryc= +github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48/go.mod h1:CRnsxgy5G8fAf5J+AM0yrsSdxXHKkIYOaq2sm+Q4DYc= +github.com/rqlite/rqlite/v8 v8.23.4 h1:DK6/7u0RkGhJvDTMQ+Ta3SB+2fd62v0RUq3aDNumWmM= +github.com/rqlite/rqlite/v8 v8.23.4/go.mod h1:TlIWBD2OvzfuPzuG7SWXVS/N9suOHfSYRNmulYKF3ds= +github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd h1:wW6BtayFoKaaDeIvXRE3SZVPOscSKlYD+X3bB749+zk= +github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg= +github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= +github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgjeY= +github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o= +github.com/txix-open/etp/v3 v3.1.0 h1:EQdjm0x2K6Dr6W8oRvs/dSU4rnw+g9o1THahIxLBC3E= +github.com/txix-open/etp/v3 v3.1.0/go.mod h1:aXqeKBdu55VOtmzx+PMlDAcz6eTNBMt3vP3NkiE1qj0= +github.com/txix-open/grmq v1.6.0 h1:V4QRb0sI2CMfdMaMsFGvGvUIDGad5vr3khNU7A3LlP8= +github.com/txix-open/grmq v1.6.0/go.mod h1:s+NNNv42+32yLpjvl2tpqtyBbMB9e7kKFee3lfa/dyo= +github.com/txix-open/isp-kit v1.31.2 h1:qt6Om1wF+vpbIz01jQix9L20h9ftwn3/SasgO1RT9T0= +github.com/txix-open/isp-kit v1.31.2/go.mod h1:t6/NHdK5JvpIiQU4+NUjky4jLbo7Bz/4hzpoBxZt64U= +github.com/txix-open/jsonschema v1.2.0 h1:B8TrdSsPhDvYv67oi/LVqf7/+PuqFlMsNwUzGeHfl1s= +github.com/txix-open/jsonschema v1.2.0/go.mod h1:l8YDZ1nvJrw6uxWowSVOxCV/ebiMJyapffW87ZEqH00= +github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= +github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= +go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.51.0 h1:974XTyIwHI4nHa1+uSLxHtUnlJ2DiVtAJjk7fd07p/8= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.51.0/go.mod h1:ZvX/taFlN6TGaOOM6D42wrNwPKUV1nGO2FuUXkityBU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= +go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU= +google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= +modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= +modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= +nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= From 80a99c425bbcc9154a0fdb647620ff16d369e77b Mon Sep 17 00:00:00 2001 From: d1slike Date: Tue, 7 May 2024 20:16:44 +0300 Subject: [PATCH 06/40] Implement event stream --- assembly/locator.go | 60 ++++-- controller/module.go | 9 +- entity/backend.go | 1 + entity/event.go | 19 +- go.mod | 6 - go.sum | 21 -- helpers/helpers.go | 1 - migrations/20240503120009_init.sql | 2 +- repository/backend.go | 39 +++- repository/config.go | 2 +- repository/config_schema.go | 2 +- repository/event.go | 9 +- repository/module.go | 34 +++- service/event/cleaner.go | 35 ++++ service/event/compactor.go | 27 +++ service/event/handler.go | 73 +++++++ service/event/worker.go | 66 +++++++ service/{module.go => module/service.go} | 80 +++++--- service/rqlite/db/db.go | 26 +-- service/rqlite/db/iface.go | 13 ++ service/rqlite/goose_store/store.go | 3 +- service/startup/service.go | 29 ++- service/subscription/rooms.go | 17 ++ service/subscription/service.go | 236 +++++++++++++++++++++++ 24 files changed, 697 insertions(+), 113 deletions(-) create mode 100644 service/event/cleaner.go create mode 100644 service/event/compactor.go create mode 100644 service/event/handler.go create mode 100644 service/event/worker.go rename service/{module.go => module/service.go} (73%) create mode 100644 service/rqlite/db/iface.go create mode 100644 service/subscription/rooms.go create mode 100644 service/subscription/service.go diff --git a/assembly/locator.go b/assembly/locator.go index c2abdbe..0ed28b7 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -2,20 +2,27 @@ package assembly import ( "net/http" + "time" "github.com/txix-open/etp/v3" - "github.com/txix-open/isp-kit/db" "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/grpc/endpoint" "github.com/txix-open/isp-kit/log" + "github.com/txix-open/isp-kit/worker" "isp-config-service/controller" "isp-config-service/repository" "isp-config-service/routes" - "isp-config-service/service" + "isp-config-service/service/event" + "isp-config-service/service/module" + "isp-config-service/service/rqlite/db" + "isp-config-service/service/subscription" ) const ( - wsReadLimit = 4 * 1024 * 1024 + wsReadLimit = 4 * 1024 * 1024 + handleEventsInterval = 500 * time.Millisecond + cleanEventInterval = 60 * time.Second + eventTtl = 60 * time.Second ) type Locator struct { @@ -31,9 +38,11 @@ func NewLocator(logger log.Logger, db db.DB) Locator { } type Config struct { - GrpcMux *grpc.Mux - HttpMux http.Handler - EtpSrv *etp.Server + GrpcMux *grpc.Mux + HttpMux http.Handler + EtpSrv *etp.Server + HandleEventWorker *worker.Worker + CleanEventWorker *worker.Worker } func (l Locator) Config() Config { @@ -42,12 +51,28 @@ func (l Locator) Config() Config { eventRepo := repository.NewEvent(l.db) configRepo := repository.NewConfig(l.db) configSchemaRepo := repository.NewConfigSchema(l.db) - moduleService := service.NewModule( + + etpSrv := etp.NewServer( + etp.WithServerReadLimit(wsReadLimit), + etp.WithServerAcceptOptions(&etp.AcceptOptions{ + InsecureSkipVerify: true, + }), + ) + subscriptionService := subscription.NewService( + moduleRepo, + backendRepo, + configRepo, + etpSrv.Rooms(), + l.logger, + ) + + moduleService := module.NewService( moduleRepo, backendRepo, eventRepo, configRepo, configSchemaRepo, + subscriptionService, l.logger, ) moduleController := controller.NewModule(moduleService, l.logger) @@ -57,19 +82,22 @@ func (l Locator) Config() Config { mapper := endpoint.DefaultWrapper(l.logger) grpcMux := routes.GrpcHandler(mapper, controllers) - etpSrv := etp.NewServer( - etp.WithServerReadLimit(wsReadLimit), - etp.WithServerAcceptOptions(&etp.AcceptOptions{ - InsecureSkipVerify: true, - }), - ) routes.BindEtp(etpSrv, controllers) httpMux := routes.HttpHandler(etpSrv) + eventHandler := event.NewHandler(subscriptionService, l.logger) + handleEventJob := event.NewWorker(eventRepo, eventHandler, l.logger) + handleEventWorker := worker.New(handleEventJob, worker.WithInterval(handleEventsInterval)) + + cleanerJob := event.NewCleaner(eventRepo, eventTtl, l.logger) + cleanEventWorker := worker.New(cleanerJob, worker.WithInterval(cleanEventInterval)) + return Config{ - GrpcMux: grpcMux, - EtpSrv: etpSrv, - HttpMux: httpMux, + GrpcMux: grpcMux, + EtpSrv: etpSrv, + HttpMux: httpMux, + HandleEventWorker: handleEventWorker, + CleanEventWorker: cleanEventWorker, } } diff --git a/controller/module.go b/controller/module.go index ee2b9ab..1cab60c 100644 --- a/controller/module.go +++ b/controller/module.go @@ -51,7 +51,14 @@ func NewModule(service ModuleService, logger log.Logger) Module { func (m Module) OnConnect(conn *etp.Conn) { ctx := conn.HttpRequest().Context() - err := m.service.OnConnect(ctx, conn, helpers.ModuleName(conn)) + err := conn.HttpRequest().ParseForm() + if err != nil { + m.handleError(ctx, errors.WithMessage(err, "parse form")) + _ = conn.Close() + return + } + + err = m.service.OnConnect(ctx, conn, helpers.ModuleName(conn)) if err != nil { m.handleError(ctx, errors.WithMessage(err, "handle onConnect")) } diff --git a/entity/backend.go b/entity/backend.go index 51d1d38..845042a 100644 --- a/entity/backend.go +++ b/entity/backend.go @@ -10,6 +10,7 @@ type Backend struct { Address string `json:"address"` Version string `json:"version"` LibVersion string `json:"lib_version"` + ModuleName string `json:"module_name"` Endpoints xtypes.Json[[]cluster.EndpointDescriptor] `json:"endpoints"` RequiredModules xtypes.Json[[]cluster.ModuleDependency] `json:"required_modules"` CreatedAt xtypes.Time `json:"created_at"` diff --git a/entity/event.go b/entity/event.go index 44e006b..b8cfa36 100644 --- a/entity/event.go +++ b/entity/event.go @@ -1,6 +1,8 @@ package entity import ( + "fmt" + "isp-config-service/entity/xtypes" ) @@ -15,19 +17,30 @@ const ( type Event struct { RowId int `json:"rowid"` - Type EventType `json:"type"` Payload xtypes.Json[EventPayload] `json:"payload"` } -func NewEvent(t EventType, payload EventPayload) Event { +func NewEvent(payload EventPayload) Event { return Event{ - Type: t, Payload: xtypes.Json[EventPayload]{ Value: payload, }, } } +func (e Event) Key() string { + switch { + case e.Payload.Value.ConfigUpdated != nil: + return fmt.Sprintf("config_updated_%s", e.Payload.Value.ConfigUpdated.ModuleId) + case e.Payload.Value.ModuleReady != nil: + return fmt.Sprintf("module_lifecycle_%s", e.Payload.Value.ModuleReady.ModuleId) + case e.Payload.Value.ModuleDisconnected != nil: + return fmt.Sprintf("module_lifecycle_%s", e.Payload.Value.ModuleDisconnected.ModuleId) + default: + return "" + } +} + type EventPayload struct { ConfigUpdated *PayloadConfigUpdated `json:",omitempty"` ModuleReady *PayloadModuleReady `json:",omitempty"` diff --git a/go.mod b/go.mod index fb9cddf..24cdbb9 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,6 @@ go 1.22 require ( github.com/Masterminds/squirrel v1.5.4 github.com/google/uuid v1.6.0 - github.com/iancoleman/strcase v0.3.0 - github.com/jmoiron/sqlx v1.4.0 github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.20.0 github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 @@ -39,10 +37,6 @@ require ( github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/raft v1.6.1 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect - github.com/jackc/pgx/v5 v5.5.5 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect diff --git a/go.sum b/go.sum index 18095f5..e6dc68b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= @@ -59,8 +57,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA= github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= -github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -104,18 +100,6 @@ github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7H github.com/hashicorp/raft v1.6.1 h1:v/jm5fcYHvVkL0akByAp+IDdDSzCNCGhdO6VdB56HIM= github.com/hashicorp/raft v1.6.1/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= -github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= -github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= -github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -139,8 +123,6 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhR github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -152,8 +134,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= @@ -229,7 +209,6 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= diff --git a/helpers/helpers.go b/helpers/helpers.go index ef77860..3416ec4 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -5,6 +5,5 @@ import ( ) func ModuleName(conn *etp.Conn) string { - _ = conn.HttpRequest().ParseForm() return conn.HttpRequest().Form.Get("module_name") } diff --git a/migrations/20240503120009_init.sql b/migrations/20240503120009_init.sql index fb2555d..adb5410 100644 --- a/migrations/20240503120009_init.sql +++ b/migrations/20240503120009_init.sql @@ -54,6 +54,7 @@ create table isp_config_service__backend address text not null, version text not null, lib_version text not null, + module_name text not null, endpoints blob not null, required_modules blob not null, created_at integer not null default (unixepoch()), @@ -64,7 +65,6 @@ create table isp_config_service__backend create table isp_config_service__event ( - type integer not null, payload blob not null, created_at integer not null default (unixepoch()) ); diff --git a/repository/backend.go b/repository/backend.go index b8a229c..09ab5f9 100644 --- a/repository/backend.go +++ b/repository/backend.go @@ -2,11 +2,12 @@ package repository import ( "context" + "fmt" "github.com/Masterminds/squirrel" "github.com/pkg/errors" - "github.com/txix-open/isp-kit/db" "isp-config-service/entity" + "isp-config-service/service/rqlite/db" ) type Backend struct { @@ -22,13 +23,13 @@ func NewBackend(db db.DB) Backend { func (r Backend) Upsert(ctx context.Context, backend entity.Backend) error { query, args, err := squirrel.Insert(Table("backend")). Columns("module_id", "address", - "version", "lib_version", + "version", "lib_version", "module_name", "endpoints", "required_modules"). Values(backend.ModuleId, backend.Address, - backend.Version, backend.LibVersion, + backend.Version, backend.LibVersion, backend.ModuleName, backend.Endpoints, backend.RequiredModules, ).Suffix(`on conflict (module_id, address) do update - set version = excluded.version, lib_version = excluded.lib_version, + set version = excluded.version, lib_version = excluded.lib_version, module_name = excluded.module_name, endpoints = excluded.endpoints, required_modules = excluded.required_modules, updated_at = unixepoch() `). @@ -62,3 +63,33 @@ func (r Backend) Delete(ctx context.Context, moduleId string, address string) er return nil } + +func (r Backend) All(ctx context.Context) ([]entity.Backend, error) { + result := make([]entity.Backend, 0) + query := fmt.Sprintf("SELECT * FROM %s order by created_at desc", Table("backend")) + err := r.db.Select(ctx, &result, query) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + return result, nil +} + +func (r Backend) GetByModuleId(ctx context.Context, moduleId string) ([]entity.Backend, error) { + query, args, err := squirrel.Select("*"). + From(Table("backend")). + Where(squirrel.Eq{ + "module_id": moduleId, + }).OrderBy("created_at desc"). + ToSql() + if err != nil { + return nil, errors.WithMessage(err, "build query") + } + + result := make([]entity.Backend, 0) + err = r.db.Select(ctx, &result, query, args...) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + + return result, nil +} diff --git a/repository/config.go b/repository/config.go index fa198df..9c5fee1 100644 --- a/repository/config.go +++ b/repository/config.go @@ -6,8 +6,8 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" - "github.com/txix-open/isp-kit/db" "isp-config-service/entity" + "isp-config-service/service/rqlite/db" ) type Config struct { diff --git a/repository/config_schema.go b/repository/config_schema.go index 46e9204..2289052 100644 --- a/repository/config_schema.go +++ b/repository/config_schema.go @@ -5,8 +5,8 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" - "github.com/txix-open/isp-kit/db" "isp-config-service/entity" + "isp-config-service/service/rqlite/db" ) type ConfigSchema struct { diff --git a/repository/event.go b/repository/event.go index 3e15aa3..2c30783 100644 --- a/repository/event.go +++ b/repository/event.go @@ -5,8 +5,9 @@ import ( "fmt" "github.com/pkg/errors" - "github.com/txix-open/isp-kit/db" "isp-config-service/entity" + "isp-config-service/entity/xtypes" + "isp-config-service/service/rqlite/db" ) type Event struct { @@ -20,8 +21,8 @@ func NewEvent(db db.DB) Event { } func (r Event) Insert(ctx context.Context, event entity.Event) error { - query := fmt.Sprintf("insert into %s (type, payload) values (?, ?)", Table("event")) - _, err := r.db.Exec(ctx, query, event.Type, event.Payload) + query := fmt.Sprintf("insert into %s (payload) values (?)", Table("event")) + _, err := r.db.Exec(ctx, query, event.Payload) if err != nil { return errors.WithMessagef(err, "exec: %s", query) } @@ -38,7 +39,7 @@ func (r Event) Get(ctx context.Context, lastRowId int, limit int) ([]entity.Even return result, nil } -func (r Event) DeleteByCreatedAt(ctx context.Context, before int) (int, error) { +func (r Event) DeleteByCreatedAt(ctx context.Context, before xtypes.Time) (int, error) { query := fmt.Sprintf("delete from %s where created_at < ?", Table("event")) result, err := r.db.Exec(ctx, query, before) if err != nil { diff --git a/repository/module.go b/repository/module.go index f7c9184..e1e2c53 100644 --- a/repository/module.go +++ b/repository/module.go @@ -6,9 +6,9 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" - "github.com/txix-open/isp-kit/db" "isp-config-service/entity" "isp-config-service/entity/xtypes" + "isp-config-service/service/rqlite/db" ) type Module struct { @@ -36,7 +36,7 @@ func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error result := make(map[string]string) err = r.db.SelectRow(ctx, &result, query, args...) if err != nil { - return "", errors.WithMessagef(err, "select : %s", query) + return "", errors.WithMessagef(err, "select: %s", query) } return result["id"], nil } @@ -62,6 +62,36 @@ func (r Module) SetDisconnectedAt( return nil } +func (r Module) GetByNames(ctx context.Context, names []string) ([]entity.Module, error) { + query, args, err := squirrel.Select("*"). + From(Table("module")). + Where(squirrel.Eq{ + "name": names, + }).OrderBy("created_at desc"). + ToSql() + if err != nil { + return nil, errors.WithMessage(err, "build query") + } + + result := make([]entity.Module, 0) + err = r.db.Select(ctx, &result, query, args...) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + + return result, nil +} + +func (r Module) GetById(ctx context.Context, id string) (*entity.Module, error) { + result := entity.Module{} + query := fmt.Sprintf("select * from %s where id = ?", Table("module")) + err := r.db.SelectRow(ctx, &result, query, id) + if err != nil { + return nil, errors.WithMessagef(err, "select row: %s", query) + } + return &result, nil +} + func Table(table string) string { return fmt.Sprintf("isp_config_service__%s", table) } diff --git a/service/event/cleaner.go b/service/event/cleaner.go new file mode 100644 index 0000000..b7e8f4e --- /dev/null +++ b/service/event/cleaner.go @@ -0,0 +1,35 @@ +package event + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/log" + "isp-config-service/entity/xtypes" +) + +type Cleaner struct { + repo Repo + eventTtl time.Duration + logger log.Logger +} + +func NewCleaner(repo Repo, eventTtl time.Duration, logger log.Logger) Cleaner { + return Cleaner{ + repo: repo, + eventTtl: eventTtl, + logger: logger, + } +} + +func (c Cleaner) Do(ctx context.Context) { + deleteBefore := time.Now().Add(-c.eventTtl) + deleted, err := c.repo.DeleteByCreatedAt(ctx, xtypes.Time{Value: deleteBefore}) + if err != nil { + c.logger.Error(ctx, errors.WithMessage(err, "delete old events")) + return + } + c.logger.Debug(ctx, fmt.Sprintf("delete %d old events", deleted)) +} diff --git a/service/event/compactor.go b/service/event/compactor.go new file mode 100644 index 0000000..b7ef273 --- /dev/null +++ b/service/event/compactor.go @@ -0,0 +1,27 @@ +package event + +import ( + "isp-config-service/entity" +) + +type Compactor struct { +} + +func NewCompactor() Compactor { + return Compactor{} +} + +func (c Compactor) Compact(events []entity.Event) []entity.Event { + uniqueEvents := make([]entity.Event, 0, len(events)) + uniqueEventKeys := make(map[string]bool, len(events)) + for i := len(events) - 1; i >= 0; i-- { + event := events[i] + key := event.Key() + if uniqueEventKeys[key] { + continue + } + uniqueEvents = append(uniqueEvents, event) + uniqueEventKeys[key] = true + } + return uniqueEvents +} diff --git a/service/event/handler.go b/service/event/handler.go new file mode 100644 index 0000000..ddc7899 --- /dev/null +++ b/service/event/handler.go @@ -0,0 +1,73 @@ +package event + +import ( + "context" + + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/log" + "isp-config-service/entity" +) + +type SubscriptionService interface { + NotifyConfigChanged(ctx context.Context, moduleId string) error + NotifyBackendsChanged(ctx context.Context, moduleId string) error + NotifyRoutingChanged(ctx context.Context) error +} + +type Handler struct { + subscriptionService SubscriptionService + logger log.Logger +} + +func NewHandler( + subscriptionService SubscriptionService, + logger log.Logger, +) Handler { + return Handler{ + subscriptionService: subscriptionService, + logger: logger, + } +} + +func (h Handler) Handle(ctx context.Context, events []entity.Event) { + routingChanged := false + for _, event := range events { + payload := event.Payload.Value + routingChanged = routingChanged || payload.ModuleReady != nil || payload.ModuleDisconnected != nil + err := h.handleEvent(ctx, event) + if err != nil { + h.logger.Error(ctx, errors.WithMessagef(err, "handle event: %s", event.Key())) + } + } + + if routingChanged { + err := h.subscriptionService.NotifyRoutingChanged(ctx) + if err != nil { + h.logger.Error(ctx, errors.WithMessage(err, "handle routing changed")) + } + } +} + +func (h Handler) handleEvent(ctx context.Context, event entity.Event) error { + payload := event.Payload.Value + switch { + case payload.ConfigUpdated != nil: + err := h.subscriptionService.NotifyConfigChanged(ctx, payload.ConfigUpdated.ModuleId) + if err != nil { + return errors.WithMessage(err, "handle config changed event") + } + case payload.ModuleReady != nil: + err := h.subscriptionService.NotifyBackendsChanged(ctx, payload.ModuleReady.ModuleId) + if err != nil { + return errors.WithMessage(err, "handle module ready event") + } + case payload.ModuleDisconnected != nil: + err := h.subscriptionService.NotifyBackendsChanged(ctx, payload.ModuleDisconnected.ModuleId) + if err != nil { + return errors.WithMessage(err, "handle module disconnected event") + } + default: + return errors.Errorf("unknown event: %v", payload) + } + return nil +} diff --git a/service/event/worker.go b/service/event/worker.go new file mode 100644 index 0000000..005e6cf --- /dev/null +++ b/service/event/worker.go @@ -0,0 +1,66 @@ +package event + +import ( + "context" + + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/log" + "isp-config-service/entity" + "isp-config-service/entity/xtypes" + "isp-config-service/service/rqlite/db" +) + +const ( + limit = 100 +) + +type HandlerRepo interface { + Handle(ctx context.Context, events []entity.Event) +} + +type Repo interface { + Get(ctx context.Context, lastRowId int, limit int) ([]entity.Event, error) + DeleteByCreatedAt(ctx context.Context, before xtypes.Time) (int, error) +} + +type Worker struct { + repo Repo + compactor Compactor + handler HandlerRepo + lastEventId int + logger log.Logger +} + +func NewWorker(repo Repo, handler HandlerRepo, logger log.Logger) *Worker { + return &Worker{ + repo: repo, + handler: handler, + compactor: NewCompactor(), + logger: logger, + } +} + +func (w *Worker) Do(ctx context.Context) { + ctx = log.ToContext(ctx, log.String("worker", "eventReader")) + err := w.do(ctx) + if err != nil { + w.logger.Error(ctx, err) + } +} + +func (w *Worker) do(ctx context.Context) error { + getEventCtx := db.NoneConsistency().ToContext(ctx) + events, err := w.repo.Get(getEventCtx, w.lastEventId, limit) + if err != nil { + return errors.WithMessage(err, "get new events") + } + if len(events) == 0 { + return nil + } + + events = w.compactor.Compact(events) + w.handler.Handle(ctx, events) + w.lastEventId = events[len(events)-1].RowId + + return nil +} diff --git a/service/module.go b/service/module/service.go similarity index 73% rename from service/module.go rename to service/module/service.go index a00d85e..7d57a24 100644 --- a/service/module.go +++ b/service/module/service.go @@ -1,4 +1,4 @@ -package service +package module import ( "context" @@ -23,7 +23,7 @@ const ( backendKey = "backend" ) -type ModuleRepo interface { +type Repo interface { Upsert(ctx context.Context, module entity.Module) (string, error) SetDisconnectedAt( ctx context.Context, @@ -50,34 +50,43 @@ type ConfigRepo interface { GetActive(ctx context.Context, moduleId string) (*entity.Config, error) } -type Module struct { - moduleRepo ModuleRepo - backendRepo BackendRepo - eventRepo EventRepo - configRepo ConfigRepo - configSchemaRepo ConfigSchemaRepo - logger log.Logger +type SubscriptionService interface { + SubscribeToConfigChanges(conn *etp.Conn, moduleId string) + SubscribeToBackendsChanges(ctx context.Context, conn *etp.Conn, requiredModuleNames []string) error + SubscribeToRoutingChanges(ctx context.Context, conn *etp.Conn) error } -func NewModule( - moduleRepo ModuleRepo, +type Service struct { + moduleRepo Repo + backendRepo BackendRepo + eventRepo EventRepo + configRepo ConfigRepo + configSchemaRepo ConfigSchemaRepo + subscriptionService SubscriptionService + logger log.Logger +} + +func NewService( + moduleRepo Repo, backendRepo BackendRepo, eventRepo EventRepo, configRepo ConfigRepo, configSchemaRepo ConfigSchemaRepo, + subscriptionService SubscriptionService, logger log.Logger, -) Module { - return Module{ - moduleRepo: moduleRepo, - backendRepo: backendRepo, - eventRepo: eventRepo, - configRepo: configRepo, - configSchemaRepo: configSchemaRepo, - logger: logger, +) Service { + return Service{ + moduleRepo: moduleRepo, + backendRepo: backendRepo, + eventRepo: eventRepo, + configRepo: configRepo, + configSchemaRepo: configSchemaRepo, + subscriptionService: subscriptionService, + logger: logger, } } -func (s Module) OnConnect(ctx context.Context, conn *etp.Conn, moduleName string) error { +func (s Service) OnConnect(ctx context.Context, conn *etp.Conn, moduleName string) error { now := now() module := entity.Module{ Id: uuid.NewString(), @@ -94,7 +103,7 @@ func (s Module) OnConnect(ctx context.Context, conn *etp.Conn, moduleName string return nil } -func (s Module) OnDisconnect( +func (s Service) OnDisconnect( ctx context.Context, conn *etp.Conn, moduleName string, @@ -128,7 +137,7 @@ func (s Module) OnDisconnect( return errors.WithMessage(err, "delete backend in store") } - event := entity.NewEvent(entity.ModuleDisconnected, entity.EventPayload{ + event := entity.NewEvent(entity.EventPayload{ ModuleDisconnected: &entity.PayloadModuleDisconnected{ ModuleId: moduleId, }, @@ -142,7 +151,7 @@ func (s Module) OnDisconnect( return nil } -func (s Module) OnError(ctx context.Context, conn *etp.Conn, moduleName string, err error) { +func (s Service) OnError(ctx context.Context, conn *etp.Conn, moduleName string, err error) { err = errors.WithMessagef( err, "unexpected error in communication, module: '%s'", @@ -151,7 +160,7 @@ func (s Module) OnError(ctx context.Context, conn *etp.Conn, moduleName string, s.logger.Error(ctx, err) } -func (s Module) OnModuleReady( +func (s Service) OnModuleReady( ctx context.Context, conn *etp.Conn, declaration cluster.BackendDeclaration, @@ -166,6 +175,7 @@ func (s Module) OnModuleReady( Address: fmt.Sprintf("%s:%s", declaration.Address.IP, declaration.Address.Port), Version: declaration.Version, LibVersion: declaration.LibVersion, + ModuleName: declaration.ModuleName, Endpoints: xtypes.Json[[]cluster.EndpointDescriptor]{Value: declaration.Endpoints}, RequiredModules: xtypes.Json[[]cluster.ModuleDependency]{Value: declaration.RequiredModules}, } @@ -176,7 +186,7 @@ func (s Module) OnModuleReady( conn.Data().Set(backendKey, backend) - event := entity.NewEvent(entity.ModuleReady, entity.EventPayload{ + event := entity.NewEvent(entity.EventPayload{ ModuleReady: &entity.PayloadModuleReady{ ModuleId: moduleId, }, @@ -189,15 +199,29 @@ func (s Module) OnModuleReady( return nil } -func (s Module) OnModuleRequirements( +func (s Service) OnModuleRequirements( ctx context.Context, conn *etp.Conn, requirements cluster.ModuleRequirements, ) error { + if len(requirements.RequiredModules) > 0 { + err := s.subscriptionService.SubscribeToBackendsChanges(ctx, conn, requirements.RequiredModules) + if err != nil { + return errors.WithMessage(err, "subscribe to required modules") + } + } + + if requirements.RequireRoutes { + err := s.subscriptionService.SubscribeToRoutingChanges(ctx, conn) + if err != nil { + return errors.WithMessage(err, "subscribe to routing changes") + } + } + return nil } -func (s Module) OnModuleConfigSchema( +func (s Service) OnModuleConfigSchema( ctx context.Context, conn *etp.Conn, data cluster.ConfigData, @@ -234,6 +258,8 @@ func (s Module) OnModuleConfigSchema( return errors.WithMessage(err, "send event with config") } + s.subscriptionService.SubscribeToConfigChanges(conn, moduleId) + configSchema := entity.ConfigSchema{ Id: uuid.NewString(), ModuleId: moduleId, diff --git a/service/rqlite/db/db.go b/service/rqlite/db/db.go index 8c3a912..8c67bac 100644 --- a/service/rqlite/db/db.go +++ b/service/rqlite/db/db.go @@ -4,25 +4,19 @@ import ( "context" "database/sql" - "github.com/iancoleman/strcase" - "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/tidwall/gjson" "github.com/txix-open/isp-kit/http/httpcli" "github.com/txix-open/isp-kit/json" ) -func init() { - sqlx.NameMapper = strcase.ToSnake -} - -type DB struct { +type Adapter struct { cli *httpcli.Client } -func Open(ctx context.Context, dsn string, client *httpcli.Client) (*DB, error) { +func Open(ctx context.Context, dsn string, client *httpcli.Client) (*Adapter, error) { client.GlobalRequestConfig().BaseUrl = dsn - db := &DB{ + db := &Adapter{ cli: client, } @@ -35,7 +29,7 @@ func Open(ctx context.Context, dsn string, client *httpcli.Client) (*DB, error) return db, nil } -func (d DB) Select(ctx context.Context, ptr any, query string, args ...any) error { +func (d Adapter) Select(ctx context.Context, ptr any, query string, args ...any) error { result := &Result{ Rows: ptr, } @@ -62,7 +56,7 @@ func (d DB) Select(ctx context.Context, ptr any, query string, args ...any) erro return nil } -func (d DB) SelectRow(ctx context.Context, ptr any, query string, args ...any) error { +func (d Adapter) SelectRow(ctx context.Context, ptr any, query string, args ...any) error { result := &Result{} resp := Response{ Results: []*Result{result}, @@ -101,7 +95,7 @@ func (d DB) SelectRow(ctx context.Context, ptr any, query string, args ...any) e return nil } -func (d DB) Exec(ctx context.Context, query string, args ...any) (sql.Result, error) { +func (d Adapter) Exec(ctx context.Context, query string, args ...any) (sql.Result, error) { result := &Result{} resp := Response{ Results: []*Result{result}, @@ -122,10 +116,6 @@ func (d DB) Exec(ctx context.Context, query string, args ...any) (sql.Result, er return result, nil } -func (d DB) ExecNamed(ctx context.Context, query string, arg any) (sql.Result, error) { - query, args, err := sqlx.Named(query, arg) - if err != nil { - return nil, errors.WithMessage(err, "map to unnamed query") - } - return d.Exec(ctx, query, args...) +func (d Adapter) ExecNamed(ctx context.Context, query string, arg any) (sql.Result, error) { + return d.Exec(ctx, query, arg) } diff --git a/service/rqlite/db/iface.go b/service/rqlite/db/iface.go new file mode 100644 index 0000000..1626c76 --- /dev/null +++ b/service/rqlite/db/iface.go @@ -0,0 +1,13 @@ +package db + +import ( + "context" + "database/sql" +) + +type DB interface { + Select(ctx context.Context, ptr any, query string, args ...any) error + SelectRow(ctx context.Context, ptr any, query string, args ...any) error + Exec(ctx context.Context, query string, args ...any) (sql.Result, error) + ExecNamed(ctx context.Context, query string, arg any) (sql.Result, error) +} diff --git a/service/rqlite/goose_store/store.go b/service/rqlite/goose_store/store.go index e78eae4..4b9b254 100644 --- a/service/rqlite/goose_store/store.go +++ b/service/rqlite/goose_store/store.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/pressly/goose/v3/database" + "isp-config-service/repository" ) type Store struct { @@ -18,7 +19,7 @@ type Store struct { func NewStore(db *sql.DB) Store { return Store{ db: db, - tablename: "isp_config_service__goose_db_version", + tablename: repository.Table("goose_db_version"), querier: &Rqlite{}, } } diff --git a/service/startup/service.go b/service/startup/service.go index f47e5bd..bf9c516 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -17,6 +17,7 @@ import ( "github.com/txix-open/isp-kit/http/httpcli" "github.com/txix-open/isp-kit/http/httpclix" "github.com/txix-open/isp-kit/log" + "github.com/txix-open/isp-kit/worker" "isp-config-service/assembly" "isp-config-service/conf" "isp-config-service/service/rqlite" @@ -37,12 +38,14 @@ type Service struct { logger log.Logger //initialized in Run - etpSrv *etp.Server + etpSrv *etp.Server + handleEventWorker *worker.Worker + cleanEventWorker *worker.Worker } -func New(boot *bootstrap.Bootstrap) Service { +func New(boot *bootstrap.Bootstrap) *Service { rqlite := rqlite.New(boot.App.Config()) - return Service{ + return &Service{ boot: boot, rqlite: rqlite, grpcSrv: grpc.DefaultServer(), @@ -52,7 +55,7 @@ func New(boot *bootstrap.Bootstrap) Service { } } -func (s Service) Run(ctx context.Context) error { +func (s *Service) Run(ctx context.Context) error { localConfig := conf.Local{} err := s.boot.App.Config().Read(&localConfig) if err != nil { @@ -91,9 +94,14 @@ func (s Service) Run(ctx context.Context) error { config := assembly.NewLocator(s.logger, db).Config() s.etpSrv = config.EtpSrv + s.handleEventWorker = config.HandleEventWorker + s.cleanEventWorker = config.CleanEventWorker s.grpcSrv.Upgrade(config.GrpcMux) s.httpSrv.Upgrade(config.HttpMux) + s.handleEventWorker.Run(ctx) + s.cleanEventWorker.Run(ctx) + go func() { s.logger.Debug(ctx, fmt.Sprintf("starting grpc server on %s", s.boot.BindingAddress)) err := s.grpcSrv.ListenAndServe(s.boot.BindingAddress) @@ -122,7 +130,7 @@ func (s Service) Run(ctx context.Context) error { return nil } -func (s Service) Closers() []app.Closer { +func (s *Service) Closers() []app.Closer { return []app.Closer{ app.CloserFunc(func() error { s.grpcSrv.Shutdown() @@ -133,6 +141,15 @@ func (s Service) Closers() []app.Closer { defer cancel() return s.httpSrv.Shutdown(ctx) }), + app.CloserFunc(func() error { + if s.handleEventWorker != nil { + s.handleEventWorker.Shutdown() + } + if s.cleanEventWorker != nil { + s.cleanEventWorker.Shutdown() + } + return nil + }), app.CloserFunc(func() error { if s.etpSrv == nil { return nil @@ -145,7 +162,7 @@ func (s Service) Closers() []app.Closer { } } -func (s Service) leaderStartup(ctx context.Context) error { +func (s *Service) leaderStartup(ctx context.Context) error { db, err := s.rqlite.SqlDB() if err != nil { return errors.WithMessage(err, "open sql db") diff --git a/service/subscription/rooms.go b/service/subscription/rooms.go new file mode 100644 index 0000000..8b1d9f1 --- /dev/null +++ b/service/subscription/rooms.go @@ -0,0 +1,17 @@ +package subscription + +import ( + "fmt" +) + +func ConfigChangingRoom(moduleId string) string { + return fmt.Sprintf("config_changing_room.%s", moduleId) +} + +func BackendsChangingRoom(moduleId string) string { + return fmt.Sprintf("backends_changing_room.%s", moduleId) +} + +func RoutingChangingRoom() string { + return "routing_changing_room" +} diff --git a/service/subscription/service.go b/service/subscription/service.go new file mode 100644 index 0000000..fe45bcd --- /dev/null +++ b/service/subscription/service.go @@ -0,0 +1,236 @@ +package subscription + +import ( + "context" + "net" + "time" + + "github.com/pkg/errors" + "github.com/txix-open/etp/v3" + "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/json" + "github.com/txix-open/isp-kit/log" + "isp-config-service/entity" + "isp-config-service/helpers" + "isp-config-service/service/rqlite/db" +) + +type ModuleRepo interface { + GetByNames(ctx context.Context, names []string) ([]entity.Module, error) + GetById(ctx context.Context, id string) (*entity.Module, error) +} + +type BackendRepo interface { + GetByModuleId(ctx context.Context, moduleId string) ([]entity.Backend, error) + All(ctx context.Context) ([]entity.Backend, error) +} + +type ConfigRepo interface { + GetActive(ctx context.Context, moduleId string) (*entity.Config, error) +} + +type Service struct { + moduleRepo ModuleRepo + backendRepo BackendRepo + configRepo ConfigRepo + rooms *etp.Rooms + logger log.Logger +} + +func NewService( + moduleRepo ModuleRepo, + backendRepo BackendRepo, + configRepo ConfigRepo, + rooms *etp.Rooms, + logger log.Logger, +) Service { + return Service{ + moduleRepo: moduleRepo, + backendRepo: backendRepo, + configRepo: configRepo, + rooms: rooms, + logger: logger, + } +} + +func (s Service) SubscribeToConfigChanges(conn *etp.Conn, moduleId string) { + s.rooms.Join(conn, ConfigChangingRoom(moduleId)) +} + +func (s Service) NotifyConfigChanged(ctx context.Context, moduleId string) error { + conns := s.rooms.ToBroadcast(ConfigChangingRoom(moduleId)) + if len(conns) == 0 { + return nil + } + + ctx = db.NoneConsistency().ToContext(ctx) + config, err := s.configRepo.GetActive(ctx, moduleId) + if err != nil { + return errors.WithMessage(err, "get active config") + } + + for _, conn := range conns { + go func() { + s.emitEvent(ctx, conn, cluster.ConfigSendConfigChanged, config.Data) + }() + } + + return nil +} + +func (s Service) SubscribeToBackendsChanges(ctx context.Context, conn *etp.Conn, requiredModuleNames []string) error { + ctx = db.NoneConsistency().ToContext(ctx) + modules, err := s.moduleRepo.GetByNames(ctx, requiredModuleNames) + if err != nil { + return errors.WithMessage(err, "get modules by names") + } + + roomsToJoin := make([]string, 0) + for _, module := range modules { + roomsToJoin = append(roomsToJoin, BackendsChangingRoom(module.Id)) + go func() { + err := s.notifyBackendsChanged(ctx, module.Id, []*etp.Conn{conn}) + if err != nil { + s.logger.Error(ctx, errors.WithMessage(err, "notify backends changed")) + } + }() + } + + s.rooms.Join(conn, roomsToJoin...) + + return nil +} + +func (s Service) NotifyBackendsChanged(ctx context.Context, moduleId string) error { + conns := s.rooms.ToBroadcast(BackendsChangingRoom(moduleId)) + if len(conns) == 0 { + return nil + } + return s.notifyBackendsChanged(ctx, moduleId, conns) +} + +func (s Service) notifyBackendsChanged( + ctx context.Context, + moduleId string, + conns []*etp.Conn, +) error { + ctx = db.NoneConsistency().ToContext(ctx) + module, err := s.moduleRepo.GetById(ctx, moduleId) + if err != nil { + return errors.WithMessage(err, "get module by id") + } + backends, err := s.backendRepo.GetByModuleId(ctx, moduleId) + if err != nil { + return errors.WithMessage(err, "get backends by module id") + } + + addresses := make([]cluster.AddressConfiguration, 0) + for _, backend := range backends { + addr, err := splitAddress(backend) + if err != nil { + return errors.WithMessage(err, "split address") + } + addresses = append(addresses, addr) + } + data, err := json.Marshal(addresses) + if err != nil { + return errors.WithMessage(err, "marshal addresses") + } + + for _, conn := range conns { + go func() { + s.emitEvent(ctx, conn, cluster.ModuleConnectedEvent(module.Name), data) + }() + } + + return nil +} + +func (s Service) SubscribeToRoutingChanges(ctx context.Context, conn *etp.Conn) error { + err := s.notifyRoutingChanged(ctx, cluster.ConfigSendRoutesWhenConnected, []*etp.Conn{conn}) + if err != nil { + return errors.WithMessage(err, "notify routing changed") + } + + s.rooms.Join(conn, RoutingChangingRoom()) + + return nil +} + +func (s Service) NotifyRoutingChanged(ctx context.Context) error { + conns := s.rooms.ToBroadcast(RoutingChangingRoom()) + if len(conns) == 0 { + return nil + } + return s.notifyRoutingChanged(ctx, cluster.ConfigSendRoutesChanged, conns) +} + +func (s Service) notifyRoutingChanged(ctx context.Context, event string, conns []*etp.Conn) error { + backends, err := s.backendRepo.All(ctx) + if err != nil { + return errors.WithMessage(err, "get all backends") + } + + routingConfig := cluster.RoutingConfig{} + for _, backend := range backends { + addr, err := splitAddress(backend) + if err != nil { + return errors.WithMessage(err, "split address") + } + declaration := cluster.BackendDeclaration{ + ModuleName: backend.ModuleName, + Version: backend.Version, + LibVersion: backend.LibVersion, + Endpoints: backend.Endpoints.Value, + RequiredModules: backend.RequiredModules.Value, + Address: addr, + } + routingConfig = append(routingConfig, declaration) + } + data, err := json.Marshal(routingConfig) + if err != nil { + return errors.WithMessage(err, "marshal routing config") + } + + for _, conn := range conns { + go func() { + s.emitEvent(ctx, conn, event, data) + }() + } + + return nil +} + +const ( + emitEventTimeout = 5 * time.Second +) + +func (s Service) emitEvent( + ctx context.Context, + conn *etp.Conn, + event string, + data []byte, +) { + ctx, cancel := context.WithTimeout(ctx, emitEventTimeout) + defer cancel() + err := conn.Emit(ctx, event, data) + if err != nil { + err := errors.WithMessagef( + err, + "emit event %s, to %s module, connId: %d", + event, helpers.ModuleName(conn), conn.Id(), + ) + s.logger.Error(ctx, err) + } +} + +func splitAddress(backend entity.Backend) (cluster.AddressConfiguration, error) { + host, port, err := net.SplitHostPort(backend.Address) + if err != nil { + return cluster.AddressConfiguration{}, errors.WithMessagef(err, "split backend %s address: %s", backend.ModuleId, backend.Address) + } + return cluster.AddressConfiguration{ + IP: host, + Port: port, + }, nil +} From b2e34c00a4a3ecc6929487f10462c2fd569e8a9a Mon Sep 17 00:00:00 2001 From: d1slike Date: Wed, 8 May 2024 15:40:14 +0300 Subject: [PATCH 07/40] Implement api --- assembly/locator.go | 16 +++- conf/config.yml | 1 + controller/api/config.go | 118 +++++++++++++++++++++++++++++ controller/api/config_history.go | 52 +++++++++++++ controller/api/config_schema.go | 36 +++++++++ controller/api/module.go | 65 ++++++++++++++++ controller/module.go | 9 ++- domain/errors.go | 5 ++ domain/request.go | 26 +++++++ domain/response.go | 61 +++++++++++++++ entity/event.go | 11 +-- entity/xtypes/bool.go | 8 +- entity/xtypes/time.go | 12 +-- helpers/helpers.go | 16 ++++ main.go | 3 +- migrations/20240503120009_init.sql | 4 +- repository/config_schema.go | 11 +++ repository/event.go | 6 +- repository/module.go | 36 +++++++-- routes/routes.go | 53 ++++++++++++- service/api/config.go | 34 +++++++++ service/api/config_history.go | 18 +++++ service/api/config_schema.go | 14 ++++ service/api/module.go | 116 ++++++++++++++++++++++++++++ service/event/cleaner.go | 2 +- service/event/compactor.go | 3 + service/event/worker.go | 2 +- service/module/service.go | 19 ++--- service/rqlite/db/db.go | 2 +- service/subscription/service.go | 18 +---- 30 files changed, 708 insertions(+), 69 deletions(-) create mode 100644 controller/api/config.go create mode 100644 controller/api/config_history.go create mode 100644 controller/api/config_schema.go create mode 100644 controller/api/module.go create mode 100644 domain/errors.go create mode 100644 domain/request.go create mode 100644 domain/response.go create mode 100644 service/api/config.go create mode 100644 service/api/config_history.go create mode 100644 service/api/config_schema.go create mode 100644 service/api/module.go diff --git a/assembly/locator.go b/assembly/locator.go index 0ed28b7..b8e3cae 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -10,8 +10,10 @@ import ( "github.com/txix-open/isp-kit/log" "github.com/txix-open/isp-kit/worker" "isp-config-service/controller" + "isp-config-service/controller/api" "isp-config-service/repository" "isp-config-service/routes" + apisvs "isp-config-service/service/api" "isp-config-service/service/event" "isp-config-service/service/module" "isp-config-service/service/rqlite/db" @@ -76,8 +78,20 @@ func (l Locator) Config() Config { l.logger, ) moduleController := controller.NewModule(moduleService, l.logger) + + moduleApiService := apisvs.NewModule(moduleRepo, backendRepo, configSchemaRepo) + moduleApiController := api.NewModule(moduleApiService) + + configApiController := api.NewConfig(nil) + configHistoryController := api.NewConfigHistory(nil) + configSchemaController := api.NewConfigSchema(nil) + controllers := routes.Controllers{ - Module: moduleController, + Module: moduleController, + ModuleApi: moduleApiController, + ConfigApi: configApiController, + ConfigHistoryApi: configHistoryController, + ConfigSchemaApi: configSchemaController, } mapper := endpoint.DefaultWrapper(l.logger) grpcMux := routes.GrpcHandler(mapper, controllers) diff --git a/conf/config.yml b/conf/config.yml index 1867d07..f0bf664 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -18,4 +18,5 @@ rqlite: RaftAddr: "127.0.0.1:9003" JoinAddrs: "127.0.0.1:9003" FKConstraints: true + AutoVacInterval: 12h diff --git a/controller/api/config.go b/controller/api/config.go new file mode 100644 index 0000000..f38d37e --- /dev/null +++ b/controller/api/config.go @@ -0,0 +1,118 @@ +package api + +import ( + "context" + + "isp-config-service/domain" +) + +type ConfigService interface { + GetActiveConfigByModuleName(ctx context.Context, moduleName string) (*domain.Config, error) + GetConfigsByModuleId(ctx context.Context, moduleId string) ([]domain.Config, error) + CreateUpdateConfig(ctx context.Context, moduleId string) (*domain.Config, error) + GetConfigById(ctx context.Context, configId string) (*domain.Config, error) + MarkConfigAsActive(ctx context.Context, configId string) error + DeleteConfigs(ctx context.Context, idList []string) error +} + +type Config struct { + service ConfigService +} + +func NewConfig(service ConfigService) Config { + return Config{ + service: service, + } +} + +// GetActiveConfigByModuleName +// @Summary Метод получения объекта конфигурации по названию модуля +// @Description Возвращает активную конфиграцию по названию модуля +// @Tags Конфигурация +// @Accept json +// @Produce json +// @Param body body domain.GetByModuleNameRequest true "название модуля" +// @Success 200 {object} domain.Config +// @Failure 404 {object} apierrors.Error "если конфигурация не найдена" +// @Failure 500 {object} apierrors.Error +// @Router /config/get_active_config_by_module_name [POST] +func (c Config) GetActiveConfigByModuleName(ctx context.Context, request domain.GetByModuleNameRequest) (*domain.Config, error) { + +} + +// GetConfigsByModuleId +// @Summary Метод получения списка конфигураций по ID модуля +// @Description Возвращает список конфиграции по ID модуля +// @Tags Конфигурация +// @Accept json +// @Produce json +// @Param body body domain.GetByModuleIdRequest true "ID модуля" +// @Success 200 {array} domain.Config +// @Failure 400 {object} apierrors.Error "если идентификатор не указан" +// @Failure 404 {object} apierrors.Error "если конфигурация не найдена" +// @Failure 500 {object} apierrors.Error +// @Router /config/get_configs_by_module_id [POST] +func (c Config) GetConfigsByModuleId(ctx context.Context, request domain.GetByModuleIdRequest) ([]domain.Config, error) { + +} + +// CreateUpdateConfig +// @Summary Метод обновления конфигурации +// @Description Если конфиг с таким id существует, то обновляет данные, если нет, то добавляет данные в базу +// @Description В случае обновления рассылает всем подключенным модулям актуальную конфигурацию +// @Tags Конфигурация +// @Accept json +// @Produce json +// @Param body body domain.CreateUpdateConfigRequest true "объект для сохранения" +// @Success 200 {object} domain.Config +// @Failure 404 {object} apierrors.Error "если конфигурация не найдена" +// @Failure 500 {object} apierrors.Error +// @Router /config/create_update_config [POST] +func (c Config) CreateUpdateConfig(ctx context.Context, config domain.CreateUpdateConfigRequest) (*domain.Config, error) { + +} + +// GetConfigById +// @Summary Метод получение актуальной конфигурации конфигурации +// @Description Возвращает актуальную версию конфигурации без дополнительного содержимого (ConfigData) +// @Tags Конфигурация +// @Accept json +// @Produce json +// @Param body body domain.ConfigIdRequest true "id конфигурации" +// @Success 200 {object} domain.Config +// @Failure 400 {object} apierrors.Error "если не указан идентификатор конфигурации" +// @Failure 500 {object} apierrors.Error +// @Router /config/get_config_by_id [POST] +func (c Config) GetConfigById(ctx context.Context, req domain.ConfigIdRequest) (domain.Config, error) { + +} + +// MarkConfigAsActive +// @Summary Метод активации конфигурации для модуля +// @Description Активирует указанную конфигурацию и деактивирует остальные, возвращает активированную конфигурацию +// @Tags Конфигурация +// @Accept json +// @Produce json +// @Param body body domain.ConfigIdRequest true "id конфигурации для изменения" +// @Success 200 {object} domain.Config "активированная конфигурация" +// @Failure 404 {object} apierrors.Error "если конфигурация не найдена" +// @Failure 500 {object} apierrors.Error +// @Router /config/mark_config_as_active [POST] +func (c Config) MarkConfigAsActive(ctx context.Context, identity domain.ConfigIdRequest) (*domain.Config, error) { + +} + +// DeleteConfigs +// @Summary Метод удаления объектов конфигурации по идентификаторам +// @Description Возвращает количество удаленных модулей +// @Tags Конфигурация +// @Accept json +// @Produce json +// @Param body body []string true "массив идентификаторов конфигураций" +// @Success 200 {object} domain.DeleteResponse +// @Failure 400 {object} apierrors.Error "если не указан массив идентификаторов" +// @Failure 500 {object} apierrors.Error +// @Router /config/delete_config [POST] +func (c Config) DeleteConfigs(ctx context.Context, identities []string) (*domain.DeleteResponse, error) { + +} diff --git a/controller/api/config_history.go b/controller/api/config_history.go new file mode 100644 index 0000000..276dd69 --- /dev/null +++ b/controller/api/config_history.go @@ -0,0 +1,52 @@ +package api + +import ( + "context" + + "isp-config-service/domain" +) + +type ConfigHistoryService interface { + GetAllVersion(ctx context.Context, configId string) ([]domain.ConfigVersion, error) + DeleteConfigVersion(ctx context.Context, id string) error +} + +type ConfigHistory struct { + service ConfigHistoryService +} + +func NewConfigHistory(service ConfigHistoryService) ConfigHistory { + return ConfigHistory{ + service: service, + } +} + +// GetAllVersion +// @Summary Метод получение старых версий конфигурации +// @Description Возвращает предыдущие версии конфигураций +// @Tags Конфигурация +// @Accept json +// @Produce json +// @Param body body domain.ConfigIdRequest true "id конфигурации" +// @Success 200 {array} domain.ConfigVersion +// @Failure 400 {object} apierrors.Error "если не указан массив идентификаторов" +// @Failure 500 {object} apierrors.Error +// @Router /config/get_all_version [POST] +func (c ConfigHistory) GetAllVersion(ctx context.Context, req domain.ConfigIdRequest) ([]domain.ConfigVersion, error) { + +} + +// DeleteConfigVersion +// @Summary Метод удаления версии конфигурации +// @Description Возвращает количество удаленных версий +// @Tags Конфигурация +// @Accept json +// @Produce json +// @Param body body domain.ConfigIdRequest true "id версии конфигурации" +// @Success 200 {object} domain.DeleteResponse +// @Failure 400 {object} apierrors.Error "если не указан массив идентификаторов" +// @Failure 500 {object} apierrors.Error +// @Router /config/delete_version [POST] +func (c ConfigHistory) DeleteConfigVersion(ctx context.Context, req domain.ConfigIdRequest) (*domain.DeleteResponse, error) { + +} diff --git a/controller/api/config_schema.go b/controller/api/config_schema.go new file mode 100644 index 0000000..94c4b70 --- /dev/null +++ b/controller/api/config_schema.go @@ -0,0 +1,36 @@ +package api + +import ( + "context" + + "isp-config-service/domain" +) + +type ConfigSchemaService interface { + SchemaByModuleId(ctx context.Context, moduleId string) (*domain.ConfigSchema, error) +} + +type ConfigSchema struct { + service ConfigSchemaService +} + +func NewConfigSchema(service ConfigSchemaService) ConfigSchema { + return ConfigSchema{ + service: service, + } +} + +// SchemaByModuleId +// @Summary Метод получения схемы конфигурации модуля +// @Description Возвращает текущую json схему конфигурации модуля +// @Tags Схема +// @Accept json +// @Produce json +// @Param body body domain.GetByModuleIdRequest true "идентификатор модуля" +// @Success 200 {object} domain.ConfigSchema +// @Failure 404 {object} apierrors.Error "если схема для модуля не найдена" +// @Failure 500 {object} apierrors.Error +// @Router /schema/get_by_module_id [POST] +func (c ConfigSchema) SchemaByModuleId(ctx context.Context, request domain.GetByModuleIdRequest) (*domain.ConfigSchema, error) { + +} diff --git a/controller/api/module.go b/controller/api/module.go new file mode 100644 index 0000000..4819c87 --- /dev/null +++ b/controller/api/module.go @@ -0,0 +1,65 @@ +package api + +import ( + "context" + + "github.com/txix-open/isp-kit/grpc/apierrors" + "isp-config-service/domain" +) + +type ModuleService interface { + Status(ctx context.Context) ([]domain.ModuleInfo, error) + Delete(ctx context.Context, idList []string) error +} + +type Module struct { + service ModuleService +} + +func NewModule(service ModuleService) Module { + return Module{ + service: service, + } +} + +// GetModulesAggregatedInfo +// @Summary Метод полчения полной информации о состоянии модулей +// @Description Возвращает полное состояние всех модулей в кластере (схема конфигурации, подключенные экземпляры) +// @Tags Модули +// @Accept json +// @Produce json +// @Success 200 {array} domain.ModuleInfo +// @Failure 500 {object} apierrors.Error +// @Router /module/get_modules_info [POST] +func (c Module) GetModulesAggregatedInfo(ctx context.Context) ([]domain.ModuleInfo, error) { + modulesInfo, err := c.service.Status(ctx) + if err != nil { + return nil, apierrors.NewInternalServiceError(err) + } + return modulesInfo, nil +} + +// DeleteModules +// @Summary Метод удаления объектов модулей по идентификаторам +// @Description Возвращает количество удаленных модулей +// @Tags Модули +// @Accept json +// @Produce json +// @Param body body []string true "массив идентификаторов модулей" +// @Success 200 {object} domain.DeleteResponse +// @Failure 500 {object} apierrors.Error +// @Router /module/delete_module [POST] +func (c Module) DeleteModules(ctx context.Context, identities []string) (*domain.DeleteResponse, error) { + if len(identities) == 0 { + return nil, apierrors.NewBusinessError(domain.ErrorCodeBadRequest, "at least one id is required", nil) + } + + err := c.service.Delete(ctx, identities) + if err != nil { + return nil, apierrors.NewInternalServiceError(err) + } + + return &domain.DeleteResponse{ + Deleted: len(identities), + }, nil +} diff --git a/controller/module.go b/controller/module.go index 1cab60c..8ea1f95 100644 --- a/controller/module.go +++ b/controller/module.go @@ -58,7 +58,14 @@ func (m Module) OnConnect(conn *etp.Conn) { return } - err = m.service.OnConnect(ctx, conn, helpers.ModuleName(conn)) + moduleName := helpers.ModuleName(conn) + if moduleName == "" { + m.handleError(ctx, errors.New("module_name is required query param")) + _ = conn.Close() + return + } + + err = m.service.OnConnect(ctx, conn, moduleName) if err != nil { m.handleError(ctx, errors.WithMessage(err, "handle onConnect")) } diff --git a/domain/errors.go b/domain/errors.go new file mode 100644 index 0000000..ffe1036 --- /dev/null +++ b/domain/errors.go @@ -0,0 +1,5 @@ +package domain + +const ( + ErrorCodeBadRequest = 400 +) diff --git a/domain/request.go b/domain/request.go new file mode 100644 index 0000000..fc973cc --- /dev/null +++ b/domain/request.go @@ -0,0 +1,26 @@ +package domain + +import ( + "github.com/txix-open/isp-kit/json" +) + +type ConfigIdRequest struct { + Id string `validate:"required"` +} + +type GetByModuleIdRequest struct { + ModuleId string `validate:"required"` +} + +type GetByModuleNameRequest struct { + ModuleName string `validate:"required"` +} + +type CreateUpdateConfigRequest struct { + Id string + Name string `validate:"required"` + ModuleId string `validate:"required"` + Version int + Data json.RawMessage + Unsafe bool +} diff --git a/domain/response.go b/domain/response.go new file mode 100644 index 0000000..f4d22c8 --- /dev/null +++ b/domain/response.go @@ -0,0 +1,61 @@ +package domain + +import ( + "time" + + "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/json" + "github.com/txix-open/isp-kit/rc/schema" +) + +type DeleteResponse struct { + Deleted int +} + +type ModuleInfo struct { + Id string + Name string + Active bool + LastConnectedAt *time.Time + LastDisconnectedAt *time.Time + ConfigSchema json.RawMessage + Status []Connection + CreatedAt time.Time +} + +type Connection struct { + LibVersion string + Version string + Address cluster.AddressConfiguration + Endpoints []cluster.EndpointDescriptor + EstablishedAt time.Time +} + +type Config struct { + Id string + Name string + ModuleId string + Valid bool + Data json.RawMessage + Version int + Active bool + CreatedAt time.Time + UpdatedAt time.Time +} + +type ConfigVersion struct { + Id string + ConfigId string + ConfigVersion int32 + Data json.RawMessage + CreatedAt time.Time +} + +type ConfigSchema struct { + Id string + Version string + ModuleId string + Schema schema.Schema + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/entity/event.go b/entity/event.go index b8cfa36..9460fcb 100644 --- a/entity/event.go +++ b/entity/event.go @@ -6,17 +6,8 @@ import ( "isp-config-service/entity/xtypes" ) -type EventType int - -const ( - Unknown EventType = iota - ConfigUpdated - ModuleReady - ModuleDisconnected -) - type Event struct { - RowId int `json:"rowid"` + Id int `json:"id"` Payload xtypes.Json[EventPayload] `json:"payload"` } diff --git a/entity/xtypes/bool.go b/entity/xtypes/bool.go index 839cf3e..ee56b4a 100644 --- a/entity/xtypes/bool.go +++ b/entity/xtypes/bool.go @@ -1,8 +1,6 @@ package xtypes -type Bool struct { - Value bool -} +type Bool bool func (l Bool) MarshalJSON() ([]byte, error) { return l.MarshalText() @@ -13,13 +11,13 @@ func (l *Bool) UnmarshalJSON(data []byte) error { } func (l Bool) MarshalText() ([]byte, error) { - if l.Value { + if l { return []byte("1"), nil } return []byte("0"), nil } func (l *Bool) UnmarshalText(data []byte) error { - l.Value = string(data) == "1" + *l = string(data) == "1" return nil } diff --git a/entity/xtypes/time.go b/entity/xtypes/time.go index 316224c..6a6dc09 100644 --- a/entity/xtypes/time.go +++ b/entity/xtypes/time.go @@ -5,9 +5,7 @@ import ( "time" ) -type Time struct { - Value time.Time -} +type Time time.Time func (l Time) MarshalJSON() ([]byte, error) { return l.MarshalText() @@ -18,15 +16,19 @@ func (l *Time) UnmarshalJSON(data []byte) error { } func (l Time) MarshalText() ([]byte, error) { - value := strconv.Itoa(int(l.Value.Unix())) + t := time.Time(l) + value := strconv.Itoa(int(t.Unix())) return []byte(value), nil } func (l *Time) UnmarshalText(data []byte) error { + if string(data) == "null" { + return nil + } value, err := strconv.Atoi(string(data)) if err != nil { return err } - l.Value = time.Unix(int64(value), 0) + *l = Time(time.Unix(int64(value), 0)) return nil } diff --git a/helpers/helpers.go b/helpers/helpers.go index 3416ec4..f4760de 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -1,9 +1,25 @@ package helpers import ( + "net" + + "github.com/pkg/errors" "github.com/txix-open/etp/v3" + "github.com/txix-open/isp-kit/cluster" + "isp-config-service/entity" ) func ModuleName(conn *etp.Conn) string { return conn.HttpRequest().Form.Get("module_name") } + +func SplitAddress(backend entity.Backend) (cluster.AddressConfiguration, error) { + host, port, err := net.SplitHostPort(backend.Address) + if err != nil { + return cluster.AddressConfiguration{}, errors.WithMessagef(err, "split backend %s address: %s", backend.ModuleId, backend.Address) + } + return cluster.AddressConfiguration{ + IP: host, + Port: port, + }, nil +} diff --git a/main.go b/main.go index 8e77fa4..83d1c7a 100644 --- a/main.go +++ b/main.go @@ -19,11 +19,10 @@ var ( // @license.name GNU GPL v3.0 // @host localhost:9000 -// @BasePath /api/isp-config-service +// @BasePath /api/config //go:generate swag init --parseDependency //go:generate rm -f docs/swagger.json docs/docs.go - func main() { boot := bootstrap.New(version, conf.Remote{}, routes.EndpointDescriptors()) app := boot.App diff --git a/migrations/20240503120009_init.sql b/migrations/20240503120009_init.sql index adb5410..10cbcf2 100644 --- a/migrations/20240503120009_init.sql +++ b/migrations/20240503120009_init.sql @@ -65,8 +65,10 @@ create table isp_config_service__backend create table isp_config_service__event ( + id integer, payload blob not null, - created_at integer not null default (unixepoch()) + created_at integer not null default (unixepoch()), + primary key (id desc) ); -- +goose Down diff --git a/repository/config_schema.go b/repository/config_schema.go index 2289052..9bc5448 100644 --- a/repository/config_schema.go +++ b/repository/config_schema.go @@ -2,6 +2,7 @@ package repository import ( "context" + "fmt" "github.com/Masterminds/squirrel" "github.com/pkg/errors" @@ -37,3 +38,13 @@ func (r ConfigSchema) Upsert(ctx context.Context, schema entity.ConfigSchema) er return nil } + +func (r ConfigSchema) All(ctx context.Context) ([]entity.ConfigSchema, error) { + result := make([]entity.ConfigSchema, 0) + query := fmt.Sprintf("select * from %s order by created_at", Table("config_schema")) + err := r.db.Select(ctx, &result, query) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + return result, nil +} diff --git a/repository/event.go b/repository/event.go index 2c30783..736a484 100644 --- a/repository/event.go +++ b/repository/event.go @@ -29,10 +29,10 @@ func (r Event) Insert(ctx context.Context, event entity.Event) error { return nil } -func (r Event) Get(ctx context.Context, lastRowId int, limit int) ([]entity.Event, error) { - query := fmt.Sprintf("select * from %s where rowid > ? order by rowid desc limit ?", Table("event")) +func (r Event) Get(ctx context.Context, lastEventId int, limit int) ([]entity.Event, error) { + query := fmt.Sprintf("select * from %s where id > ? order by id desc limit ?", Table("event")) result := make([]entity.Event, 0) - err := r.db.Select(ctx, &result, query, lastRowId, limit) + err := r.db.Select(ctx, &result, query, lastEventId, limit) if err != nil { return nil, errors.WithMessagef(err, "select: %s", query) } diff --git a/repository/module.go b/repository/module.go index e1e2c53..a954afc 100644 --- a/repository/module.go +++ b/repository/module.go @@ -7,7 +7,6 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" "isp-config-service/entity" - "isp-config-service/entity/xtypes" "isp-config-service/service/rqlite/db" ) @@ -24,9 +23,8 @@ func NewModule(db db.DB) Module { func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error) { query, args, err := squirrel.Insert(Table("module")). Columns("id", "name", "last_connected_at"). - Values(module.Id, module.Name, module.LastConnectedAt). - Suffix(`on conflict (name) do update - set last_connected_at = excluded.last_connected_at`). + Values(module.Id, module.Name, squirrel.Expr("unixepoch()")). + Suffix(`on conflict (name) do update set last_connected_at = unixepoch()`). Suffix("returning id"). ToSql() if err != nil { @@ -41,13 +39,12 @@ func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error return result["id"], nil } -func (r Module) SetDisconnectedAt( +func (r Module) SetDisconnectedAtNow( ctx context.Context, moduleId string, - disconnected xtypes.Time, ) error { query, args, err := squirrel.Update(Table("module")). - Set("last_disconnected_at", disconnected). + Set("last_disconnected_at", squirrel.Expr("unixepoch()")). Where(squirrel.Eq{"id": moduleId}). ToSql() if err != nil { @@ -92,6 +89,31 @@ func (r Module) GetById(ctx context.Context, id string) (*entity.Module, error) return &result, nil } +func (r Module) All(ctx context.Context) ([]entity.Module, error) { + result := make([]entity.Module, 0) + query := fmt.Sprintf("select * from %s order by name", Table("module")) + err := r.db.Select(ctx, &result, query) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + return result, nil +} + +func (r Module) Delete(ctx context.Context, idList []string) error { + query, args, err := squirrel.Delete(Table("module")). + Where(squirrel.Eq{"id": idList}). + ToSql() + if err != nil { + return errors.WithMessage(err, "build query") + } + + _, err = r.db.Exec(ctx, query, args...) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + return nil +} + func Table(table string) string { return fmt.Sprintf("isp_config_service__%s", table) } diff --git a/routes/routes.go b/routes/routes.go index 88e309d..4e67f21 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -8,10 +8,15 @@ import ( "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/grpc/endpoint" "isp-config-service/controller" + "isp-config-service/controller/api" ) type Controllers struct { - Module controller.Module + Module controller.Module + ModuleApi api.Module + ConfigApi api.Config + ConfigHistoryApi api.ConfigHistory + ConfigSchemaApi api.ConfigSchema } func EndpointDescriptors() []cluster.EndpointDescriptor { @@ -42,5 +47,49 @@ func HttpHandler(etpSrv *etp.Server) http.Handler { } func endpointDescriptors(c Controllers) []cluster.EndpointDescriptor { - return []cluster.EndpointDescriptor{} + return []cluster.EndpointDescriptor{{ + Path: "module/get_modules_info", + Inner: true, + Handler: c.ModuleApi.GetModulesAggregatedInfo, + }, { + Path: "module/delete_module", + Inner: true, + Handler: c.ModuleApi.DeleteModules, + }, { + Path: "config/get_active_config_by_module_name", + Inner: true, + Handler: c.ConfigApi.GetActiveConfigByModuleName, + }, { + Path: "config/get_configs_by_module_id", + Inner: true, + Handler: c.ConfigApi.GetConfigsByModuleId, + }, { + Path: "config/create_update_config", + Inner: true, + Handler: c.ConfigApi.CreateUpdateConfig, + }, { + Path: "config/get_config_by_id", + Inner: true, + Handler: c.ConfigApi.GetConfigById, + }, { + Path: "config/mark_config_as_active", + Inner: true, + Handler: c.ConfigApi.MarkConfigAsActive, + }, { + Path: "config/delete_config", + Inner: true, + Handler: c.ConfigApi.DeleteConfigs, + }, { + Path: "config/get_all_version", + Inner: true, + Handler: c.ConfigHistoryApi.GetAllVersion, + }, { + Path: "config/delete_version", + Inner: true, + Handler: c.ConfigHistoryApi.DeleteConfigVersion, + }, { + Path: "schema/get_by_module_id", + Inner: true, + Handler: c.ConfigSchemaApi.SchemaByModuleId, + }} } diff --git a/service/api/config.go b/service/api/config.go new file mode 100644 index 0000000..c7a250c --- /dev/null +++ b/service/api/config.go @@ -0,0 +1,34 @@ +package api + +import ( + "context" + + "isp-config-service/domain" +) + +type Config struct { +} + +func (c Config) GetActiveConfigByModuleName(ctx context.Context, moduleName string) (*domain.Config, error) { + +} + +func (c Config) GetConfigsByModuleId(ctx context.Context, moduleId string) ([]domain.Config, error) { + +} + +func (c Config) CreateUpdateConfig(ctx context.Context, moduleId string) (*domain.Config, error) { + +} + +func (c Config) GetConfigById(ctx context.Context, configId string) (*domain.Config, error) { + +} + +func (c Config) MarkConfigAsActive(ctx context.Context, configId string) error { + +} + +func (c Config) DeleteConfigs(ctx context.Context, idList []string) error { + +} diff --git a/service/api/config_history.go b/service/api/config_history.go new file mode 100644 index 0000000..a8942e2 --- /dev/null +++ b/service/api/config_history.go @@ -0,0 +1,18 @@ +package api + +import ( + "context" + + "isp-config-service/domain" +) + +type ConfigHistory struct { +} + +func (c ConfigHistory) GetAllVersion(ctx context.Context, configId string) ([]domain.ConfigVersion, error) { + +} + +func (c ConfigHistory) DeleteConfigVersion(ctx context.Context, id string) error { + +} diff --git a/service/api/config_schema.go b/service/api/config_schema.go new file mode 100644 index 0000000..7e6873f --- /dev/null +++ b/service/api/config_schema.go @@ -0,0 +1,14 @@ +package api + +import ( + "context" + + "isp-config-service/domain" +) + +type ConfigSchema struct { +} + +func (c ConfigSchema) SchemaByModuleId(ctx context.Context, moduleId string) (*domain.ConfigSchema, error) { + +} diff --git a/service/api/module.go b/service/api/module.go new file mode 100644 index 0000000..c2ac3c8 --- /dev/null +++ b/service/api/module.go @@ -0,0 +1,116 @@ +package api + +import ( + "context" + "time" + + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/json" + "golang.org/x/sync/errgroup" + "isp-config-service/domain" + "isp-config-service/entity" + "isp-config-service/helpers" +) + +type ModuleRepo interface { + All(ctx context.Context) ([]entity.Module, error) + Delete(ctx context.Context, idList []string) error +} + +type BackendsRepo interface { + All(ctx context.Context) ([]entity.Backend, error) +} + +type SchemaRepo interface { + All(ctx context.Context) ([]entity.ConfigSchema, error) +} + +type Module struct { + moduleRepo ModuleRepo + backendsRepo BackendsRepo + schemaRepo SchemaRepo +} + +func NewModule( + moduleRepo ModuleRepo, + backendsRepo BackendsRepo, + schemaRepo SchemaRepo, +) Module { + return Module{ + moduleRepo: moduleRepo, + backendsRepo: backendsRepo, + schemaRepo: schemaRepo, + } +} + +func (s Module) Status(ctx context.Context) ([]domain.ModuleInfo, error) { + var ( + modules []entity.Module + backends []entity.Backend + schemas []entity.ConfigSchema + err error + ) + group, ctx := errgroup.WithContext(ctx) + group.Go(func() error { + modules, err = s.moduleRepo.All(ctx) + return errors.WithMessage(err, "get all modules") + }) + group.Go(func() error { + backends, err = s.backendsRepo.All(ctx) + return errors.WithMessage(err, "get all backends") + }) + group.Go(func() error { + schemas, err = s.schemaRepo.All(ctx) + return errors.WithMessage(err, "get all schemas") + }) + err = group.Wait() + if err != nil { + return nil, errors.WithMessage(err, "wait") + } + + connections := make(map[string][]domain.Connection, len(modules)) + for _, backend := range backends { + addr, err := helpers.SplitAddress(backend) + if err != nil { + return nil, errors.WithMessage(err, "split address") + } + conn := domain.Connection{ + LibVersion: backend.LibVersion, + Version: backend.Version, + Address: addr, + Endpoints: backend.Endpoints.Value, + EstablishedAt: time.Time(backend.UpdatedAt), + } + connections[backend.ModuleId] = append(connections[backend.ModuleId], conn) + } + + schemasMap := make(map[string]json.RawMessage) + for _, schema := range schemas { + schemasMap[schema.ModuleId] = schema.Data + } + + moduleInfos := make([]domain.ModuleInfo, 0, len(modules)) + for _, module := range modules { + info := domain.ModuleInfo{ + Id: module.Id, + Name: module.Name, + Active: len(connections[module.Id]) > 0, + LastConnectedAt: (*time.Time)(module.LastConnectedAt), + LastDisconnectedAt: (*time.Time)(module.LastDisconnectedAt), + ConfigSchema: schemasMap[module.Id], + Status: connections[module.Id], + CreatedAt: time.Time(module.CreatedAt), + } + moduleInfos = append(moduleInfos, info) + } + + return moduleInfos, nil +} + +func (s Module) Delete(ctx context.Context, idList []string) error { + err := s.moduleRepo.Delete(ctx, idList) + if err != nil { + return errors.WithMessage(err, "delete modules") + } + return nil +} diff --git a/service/event/cleaner.go b/service/event/cleaner.go index b7e8f4e..4a1c20d 100644 --- a/service/event/cleaner.go +++ b/service/event/cleaner.go @@ -26,7 +26,7 @@ func NewCleaner(repo Repo, eventTtl time.Duration, logger log.Logger) Cleaner { func (c Cleaner) Do(ctx context.Context) { deleteBefore := time.Now().Add(-c.eventTtl) - deleted, err := c.repo.DeleteByCreatedAt(ctx, xtypes.Time{Value: deleteBefore}) + deleted, err := c.repo.DeleteByCreatedAt(ctx, xtypes.Time(deleteBefore)) if err != nil { c.logger.Error(ctx, errors.WithMessage(err, "delete old events")) return diff --git a/service/event/compactor.go b/service/event/compactor.go index b7ef273..ea9d604 100644 --- a/service/event/compactor.go +++ b/service/event/compactor.go @@ -1,6 +1,8 @@ package event import ( + "slices" + "isp-config-service/entity" ) @@ -23,5 +25,6 @@ func (c Compactor) Compact(events []entity.Event) []entity.Event { uniqueEvents = append(uniqueEvents, event) uniqueEventKeys[key] = true } + slices.Reverse(uniqueEvents) return uniqueEvents } diff --git a/service/event/worker.go b/service/event/worker.go index 005e6cf..51143d7 100644 --- a/service/event/worker.go +++ b/service/event/worker.go @@ -60,7 +60,7 @@ func (w *Worker) do(ctx context.Context) error { events = w.compactor.Compact(events) w.handler.Handle(ctx, events) - w.lastEventId = events[len(events)-1].RowId + w.lastEventId = events[len(events)-1].Id return nil } diff --git a/service/module/service.go b/service/module/service.go index 7d57a24..55a40db 100644 --- a/service/module/service.go +++ b/service/module/service.go @@ -5,7 +5,6 @@ import ( "crypto/md5" "encoding/hex" "fmt" - "time" "github.com/google/uuid" "github.com/pkg/errors" @@ -25,10 +24,9 @@ const ( type Repo interface { Upsert(ctx context.Context, module entity.Module) (string, error) - SetDisconnectedAt( + SetDisconnectedAtNow( ctx context.Context, moduleId string, - disconnected xtypes.Time, ) error } @@ -87,11 +85,9 @@ func NewService( } func (s Service) OnConnect(ctx context.Context, conn *etp.Conn, moduleName string) error { - now := now() module := entity.Module{ - Id: uuid.NewString(), - Name: moduleName, - LastConnectedAt: &xtypes.Time{Value: now}, + Id: uuid.NewString(), + Name: moduleName, } moduleId, err := s.moduleRepo.Upsert(ctx, module) if err != nil { @@ -121,10 +117,9 @@ func (s Service) OnDisconnect( s.logger.Error(ctx, message) } - now := now() moduleId, _ := store.Get[string](conn.Data(), moduleIdKey) if moduleId != "" { - err = s.moduleRepo.SetDisconnectedAt(ctx, moduleName, xtypes.Time{Value: now}) + err = s.moduleRepo.SetDisconnectedAtNow(ctx, moduleName) if err != nil { return errors.WithMessage(err, "update disconnected time in store") } @@ -241,7 +236,7 @@ func (s Service) OnModuleConfigSchema( ModuleId: moduleId, Data: data.Config, Version: 1, - Active: xtypes.Bool{Value: true}, + Active: xtypes.Bool(true), } err := s.configRepo.Insert(ctx, initialConfig) if err != nil { @@ -273,7 +268,3 @@ func (s Service) OnModuleConfigSchema( return nil } - -func now() time.Time { - return time.Now().UTC() -} diff --git a/service/rqlite/db/db.go b/service/rqlite/db/db.go index 8c67bac..d89cc2e 100644 --- a/service/rqlite/db/db.go +++ b/service/rqlite/db/db.go @@ -43,7 +43,7 @@ func (d Adapter) Select(ctx context.Context, ptr any, query string, args ...any) consistencyFromContext(ctx).appendParams(params) err := d.cli.Post("/db/query"). QueryParams(params). - JsonRequestBody(request(query, args...)). + JsonRequestBody(requests(request(query, args...))). JsonResponseBody(&resp). StatusCodeToError(). DoWithoutResponse(ctx) diff --git a/service/subscription/service.go b/service/subscription/service.go index fe45bcd..feea8ad 100644 --- a/service/subscription/service.go +++ b/service/subscription/service.go @@ -2,7 +2,6 @@ package subscription import ( "context" - "net" "time" "github.com/pkg/errors" @@ -126,7 +125,7 @@ func (s Service) notifyBackendsChanged( addresses := make([]cluster.AddressConfiguration, 0) for _, backend := range backends { - addr, err := splitAddress(backend) + addr, err := helpers.SplitAddress(backend) if err != nil { return errors.WithMessage(err, "split address") } @@ -173,7 +172,7 @@ func (s Service) notifyRoutingChanged(ctx context.Context, event string, conns [ routingConfig := cluster.RoutingConfig{} for _, backend := range backends { - addr, err := splitAddress(backend) + addr, err := helpers.SplitAddress(backend) if err != nil { return errors.WithMessage(err, "split address") } @@ -217,20 +216,9 @@ func (s Service) emitEvent( if err != nil { err := errors.WithMessagef( err, - "emit event %s, to %s module, connId: %d", + "emit event '%s', to %s module, connId: %d", event, helpers.ModuleName(conn), conn.Id(), ) s.logger.Error(ctx, err) } } - -func splitAddress(backend entity.Backend) (cluster.AddressConfiguration, error) { - host, port, err := net.SplitHostPort(backend.Address) - if err != nil { - return cluster.AddressConfiguration{}, errors.WithMessagef(err, "split backend %s address: %s", backend.ModuleId, backend.Address) - } - return cluster.AddressConfiguration{ - IP: host, - Port: port, - }, nil -} From c6148a1c1cad7bd74006d865157fa601cd56b05a Mon Sep 17 00:00:00 2001 From: d1slike Date: Wed, 8 May 2024 18:45:21 +0300 Subject: [PATCH 08/40] Implement api --- controller/api/config_history.go | 4 +- controller/api/module.go | 24 +++++-- domain/response.go | 5 +- entity/config.go | 5 -- entity/errors.go | 12 ++++ repository/config.go | 59 ++++++++++++++-- repository/config_history.go | 39 +++++++++++ repository/config_schema.go | 25 +++++++ repository/module.go | 11 +-- routes/routes.go | 2 +- service/api/config.go | 117 ++++++++++++++++++++++++++++++- service/api/config_history.go | 41 ++++++++++- service/api/config_schema.go | 29 +++++++- service/api/module.go | 8 ++- service/module/service.go | 2 +- service/subscription/service.go | 3 + 16 files changed, 344 insertions(+), 42 deletions(-) create mode 100644 entity/errors.go create mode 100644 repository/config_history.go diff --git a/controller/api/config_history.go b/controller/api/config_history.go index 276dd69..dcf535a 100644 --- a/controller/api/config_history.go +++ b/controller/api/config_history.go @@ -7,8 +7,8 @@ import ( ) type ConfigHistoryService interface { - GetAllVersion(ctx context.Context, configId string) ([]domain.ConfigVersion, error) - DeleteConfigVersion(ctx context.Context, id string) error + GetAllVersions(ctx context.Context, configId string) ([]domain.ConfigVersion, error) + Delete(ctx context.Context, id string) error } type ConfigHistory struct { diff --git a/controller/api/module.go b/controller/api/module.go index 4819c87..d494e0e 100644 --- a/controller/api/module.go +++ b/controller/api/module.go @@ -9,7 +9,7 @@ import ( type ModuleService interface { Status(ctx context.Context) ([]domain.ModuleInfo, error) - Delete(ctx context.Context, idList []string) error + Delete(ctx context.Context, id string) error } type Module struct { @@ -39,7 +39,7 @@ func (c Module) GetModulesAggregatedInfo(ctx context.Context) ([]domain.ModuleIn return modulesInfo, nil } -// DeleteModules +// DeleteModule // @Summary Метод удаления объектов модулей по идентификаторам // @Description Возвращает количество удаленных модулей // @Tags Модули @@ -49,12 +49,12 @@ func (c Module) GetModulesAggregatedInfo(ctx context.Context) ([]domain.ModuleIn // @Success 200 {object} domain.DeleteResponse // @Failure 500 {object} apierrors.Error // @Router /module/delete_module [POST] -func (c Module) DeleteModules(ctx context.Context, identities []string) (*domain.DeleteResponse, error) { - if len(identities) == 0 { - return nil, apierrors.NewBusinessError(domain.ErrorCodeBadRequest, "at least one id is required", nil) +func (c Module) DeleteModule(ctx context.Context, identities []string) (*domain.DeleteResponse, error) { + id, err := getSingleId(identities) + if err != nil { + return nil, err } - - err := c.service.Delete(ctx, identities) + err = c.service.Delete(ctx, id) if err != nil { return nil, apierrors.NewInternalServiceError(err) } @@ -63,3 +63,13 @@ func (c Module) DeleteModules(ctx context.Context, identities []string) (*domain Deleted: len(identities), }, nil } + +func getSingleId(identities []string) (string, error) { + if len(identities) == 0 { + return "", apierrors.NewBusinessError(domain.ErrorCodeBadRequest, "at least one id is required", nil) + } + if len(identities) > 1 { + return "", apierrors.NewBusinessError(domain.ErrorCodeBadRequest, "accept only single identity", nil) + } + return identities[0], nil +} diff --git a/domain/response.go b/domain/response.go index f4d22c8..67b5b87 100644 --- a/domain/response.go +++ b/domain/response.go @@ -5,7 +5,6 @@ import ( "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/json" - "github.com/txix-open/isp-kit/rc/schema" ) type DeleteResponse struct { @@ -46,7 +45,7 @@ type Config struct { type ConfigVersion struct { Id string ConfigId string - ConfigVersion int32 + ConfigVersion int Data json.RawMessage CreatedAt time.Time } @@ -55,7 +54,7 @@ type ConfigSchema struct { Id string Version string ModuleId string - Schema schema.Schema + Schema json.RawMessage CreatedAt time.Time UpdatedAt time.Time } diff --git a/entity/config.go b/entity/config.go index 46313d5..0382c46 100644 --- a/entity/config.go +++ b/entity/config.go @@ -1,14 +1,9 @@ package entity import ( - "github.com/pkg/errors" "isp-config-service/entity/xtypes" ) -var ( - ErrNoActiveConfig = errors.New("no active config") -) - type Config struct { Id string `json:"id"` Name string `json:"name"` diff --git a/entity/errors.go b/entity/errors.go new file mode 100644 index 0000000..02b6c98 --- /dev/null +++ b/entity/errors.go @@ -0,0 +1,12 @@ +package entity + +import ( + "github.com/pkg/errors" +) + +var ( + ErrConfigNotFound = errors.New("no active config") + ErrModuleNotFound = errors.New("module not found") + ErrSchemaNotFound = errors.New("config schema not found") + ErrConfigNotFoundOrActive = errors.New("config not found or markers as active") +) diff --git a/repository/config.go b/repository/config.go index 9c5fee1..b0ffe27 100644 --- a/repository/config.go +++ b/repository/config.go @@ -2,7 +2,6 @@ package repository import ( "context" - "database/sql" "github.com/Masterminds/squirrel" "github.com/pkg/errors" @@ -51,14 +50,60 @@ func (r Config) GetActive(ctx context.Context, moduleId string) (*entity.Config, return nil, errors.WithMessage(err, "build query") } - result := entity.Config{} - err = r.db.SelectRow(ctx, &result, query, args...) - if errors.Is(err, sql.ErrNoRows) { - return nil, entity.ErrNoActiveConfig + return selectRow[entity.Config](ctx, r.db, query, args...) +} + +func (r Config) GetByModuleId(ctx context.Context, moduleId string) ([]entity.Config, error) { + query, args, err := squirrel.Select("*"). + From(Table("config")). + Where(squirrel.Eq{ + "module_id": moduleId, + }).OrderBy("created_at desc"). + ToSql() + if err != nil { + return nil, errors.WithMessage(err, "build query") + } + + result := make([]entity.Config, 0) + err = r.db.Select(ctx, &result, query, args...) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + + return result, nil +} + +func (r Config) GetById(ctx context.Context, id string) (*entity.Config, error) { + query, args, err := squirrel.Select("*"). + From(Table("config")). + Where(squirrel.Eq{ + "id": id, + }).ToSql() + if err != nil { + return nil, errors.WithMessage(err, "build query") } + + return selectRow[entity.Config](ctx, r.db, query, args...) +} + +func (r Config) DeleteNonActiveById(ctx context.Context, id string) (bool, error) { + query, args, err := squirrel.Delete(Table("config")). + Where(squirrel.Eq{ + "id": id, + "active": "0", + }).ToSql() if err != nil { - return nil, errors.WithMessagef(err, "select row: %s", query) + return false, errors.WithMessage(err, "build query") } - return &result, nil + result, err := r.db.Exec(ctx, query, args...) + if err != nil { + return false, errors.WithMessagef(err, "exec: %s", query) + } + + affected, err := result.RowsAffected() + if err != nil { + return false, errors.WithMessage(err, "get rows affected") + } + return affected > 0, nil } diff --git a/repository/config_history.go b/repository/config_history.go new file mode 100644 index 0000000..d4b90f0 --- /dev/null +++ b/repository/config_history.go @@ -0,0 +1,39 @@ +package repository + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + "isp-config-service/entity" + "isp-config-service/service/rqlite/db" +) + +type ConfigHistory struct { + db db.DB +} + +func NewConfigHistory(db db.DB) ConfigHistory { + return ConfigHistory{ + db: db, + } +} + +func (r ConfigHistory) GetByConfigId(ctx context.Context, configId string) ([]entity.ConfigHistory, error) { + query := fmt.Sprintf(`select * from %s where config_id = ? order by version desc`, Table("config_history")) + result := make([]entity.ConfigHistory, 0) + err := r.db.Select(ctx, &result, query, configId) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + return result, nil +} + +func (r ConfigHistory) Delete(ctx context.Context, id string) error { + query := fmt.Sprintf(`delete from %s where id = ?`, Table("config_history")) + _, err := r.db.Exec(ctx, query, id) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + return nil +} diff --git a/repository/config_schema.go b/repository/config_schema.go index 9bc5448..726854e 100644 --- a/repository/config_schema.go +++ b/repository/config_schema.go @@ -2,6 +2,7 @@ package repository import ( "context" + "database/sql" "fmt" "github.com/Masterminds/squirrel" @@ -48,3 +49,27 @@ func (r ConfigSchema) All(ctx context.Context) ([]entity.ConfigSchema, error) { } return result, nil } + +func (r ConfigSchema) GetByModuleId(ctx context.Context, moduleId string) (*entity.ConfigSchema, error) { + query, args, err := squirrel.Select("*"). + From(Table("config_schema")). + Where(squirrel.Eq{"module_id": moduleId}). + ToSql() + if err != nil { + return nil, errors.WithMessage(err, "build query") + } + + return selectRow[entity.ConfigSchema](ctx, r.db, query, args...) +} + +func selectRow[T any](ctx context.Context, db db.DB, query string, args ...interface{}) (*T, error) { + var result T + err := db.SelectRow(ctx, &result, query, args...) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, errors.WithMessagef(err, "select row: %s", query) + } + return &result, nil +} diff --git a/repository/module.go b/repository/module.go index a954afc..11dc25c 100644 --- a/repository/module.go +++ b/repository/module.go @@ -80,13 +80,8 @@ func (r Module) GetByNames(ctx context.Context, names []string) ([]entity.Module } func (r Module) GetById(ctx context.Context, id string) (*entity.Module, error) { - result := entity.Module{} query := fmt.Sprintf("select * from %s where id = ?", Table("module")) - err := r.db.SelectRow(ctx, &result, query, id) - if err != nil { - return nil, errors.WithMessagef(err, "select row: %s", query) - } - return &result, nil + return selectRow[entity.Module](ctx, r.db, query, id) } func (r Module) All(ctx context.Context) ([]entity.Module, error) { @@ -99,9 +94,9 @@ func (r Module) All(ctx context.Context) ([]entity.Module, error) { return result, nil } -func (r Module) Delete(ctx context.Context, idList []string) error { +func (r Module) Delete(ctx context.Context, id string) error { query, args, err := squirrel.Delete(Table("module")). - Where(squirrel.Eq{"id": idList}). + Where(squirrel.Eq{"id": id}). ToSql() if err != nil { return errors.WithMessage(err, "build query") diff --git a/routes/routes.go b/routes/routes.go index 4e67f21..997725d 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -54,7 +54,7 @@ func endpointDescriptors(c Controllers) []cluster.EndpointDescriptor { }, { Path: "module/delete_module", Inner: true, - Handler: c.ModuleApi.DeleteModules, + Handler: c.ModuleApi.DeleteModule, }, { Path: "config/get_active_config_by_module_name", Inner: true, diff --git a/service/api/config.go b/service/api/config.go index c7a250c..7e9c0ac 100644 --- a/service/api/config.go +++ b/service/api/config.go @@ -2,19 +2,85 @@ package api import ( "context" + "time" + "github.com/pkg/errors" + "github.com/xeipuuv/gojsonschema" "isp-config-service/domain" + "isp-config-service/entity" ) +type ConfigRepo interface { + GetActive(ctx context.Context, moduleId string) (*entity.Config, error) + GetByModuleId(ctx context.Context, moduleId string) ([]entity.Config, error) + GetById(ctx context.Context, id string) (*entity.Config, error) + DeleteNonActiveById(ctx context.Context, id string) (bool, error) +} + type Config struct { + configRepo ConfigRepo + moduleRepo ModuleRepo + schemaRepo SchemaRepo +} + +func NewConfig( + configRepo ConfigRepo, + moduleRepo ModuleRepo, + schemaRepo SchemaRepo, +) Config { + return Config{ + configRepo: configRepo, + moduleRepo: moduleRepo, + schemaRepo: schemaRepo, + } } func (c Config) GetActiveConfigByModuleName(ctx context.Context, moduleName string) (*domain.Config, error) { + modules, err := c.moduleRepo.GetByNames(ctx, []string{moduleName}) + if err != nil { + return nil, errors.WithMessage(err, "get config by module name") + } + if len(modules) == 0 { + return nil, entity.ErrModuleNotFound + } + + moduleId := modules[0].Id + config, err := c.configRepo.GetActive(ctx, moduleId) + if err != nil { + return nil, errors.WithMessage(err, "get active config") + } + if config == nil { + return nil, entity.ErrConfigNotFound + } + result := configToDto(*config, nil) + return &result, nil } func (c Config) GetConfigsByModuleId(ctx context.Context, moduleId string) ([]domain.Config, error) { + configs, err := c.configRepo.GetByModuleId(ctx, moduleId) + if err != nil { + return nil, errors.WithMessage(err, "get configs by module id") + } + if len(configs) == 0 { + return make([]domain.Config, 0), nil + } + schema, err := c.schemaRepo.GetByModuleId(ctx, moduleId) + if err != nil { + return nil, errors.WithMessage(err, "get config schema by module id") + } + var jsonSchema []byte + if schema != nil { + jsonSchema = schema.Data + } + + result := make([]domain.Config, 0) + for _, config := range configs { + result = append(result, configToDto(config, jsonSchema)) + } + + return result, nil } func (c Config) CreateUpdateConfig(ctx context.Context, moduleId string) (*domain.Config, error) { @@ -22,13 +88,62 @@ func (c Config) CreateUpdateConfig(ctx context.Context, moduleId string) (*domai } func (c Config) GetConfigById(ctx context.Context, configId string) (*domain.Config, error) { + config, err := c.configRepo.GetById(ctx, configId) + if err != nil { + return nil, errors.WithMessage(err, "get config by id") + } + if config == nil { + return nil, entity.ErrConfigNotFound + } + result := configToDto(*config, nil) + return &result, nil } func (c Config) MarkConfigAsActive(ctx context.Context, configId string) error { } -func (c Config) DeleteConfigs(ctx context.Context, idList []string) error { +func (c Config) DeleteConfig(ctx context.Context, configId string) error { + deleted, err := c.configRepo.DeleteNonActiveById(ctx, configId) + if err != nil { + return errors.WithMessage(err, "delete config") + } + if !deleted { + return entity.ErrConfigNotFoundOrActive + } + return nil +} + +func configToDto(config entity.Config, schema []byte) domain.Config { + valid := true + if schema != nil { + errors, _ := validateConfig(config.Data, schema) + valid = len(errors) == 0 + } + return domain.Config{ + Id: config.Id, + Name: config.Name, + ModuleId: config.ModuleId, + Valid: valid, + Data: config.Data, + Version: config.Version, + Active: bool(config.Active), + CreatedAt: time.Time(config.CreatedAt), + UpdatedAt: time.Time(config.UpdatedAt), + } +} +func validateConfig(config []byte, schema []byte) (map[string]string, error) { + schemaLoader := gojsonschema.NewBytesLoader(schema) + configLoader := gojsonschema.NewBytesLoader(config) + result, err := gojsonschema.Validate(schemaLoader, configLoader) + if err != nil { + return nil, errors.WithMessage(err, "validate config") + } + details := map[string]string{} + for _, resultError := range result.Errors() { + details[resultError.Field()] = resultError.Description() + } + return details, nil } diff --git a/service/api/config_history.go b/service/api/config_history.go index a8942e2..a0d9816 100644 --- a/service/api/config_history.go +++ b/service/api/config_history.go @@ -2,17 +2,52 @@ package api import ( "context" + "time" + "github.com/pkg/errors" "isp-config-service/domain" + "isp-config-service/entity" ) -type ConfigHistory struct { +type ConfigHistoryRepo interface { + Delete(ctx context.Context, id string) error + GetByConfigId(ctx context.Context, configId string) ([]entity.ConfigHistory, error) } -func (c ConfigHistory) GetAllVersion(ctx context.Context, configId string) ([]domain.ConfigVersion, error) { +type ConfigHistory struct { + repo ConfigHistoryRepo +} +func NewConfigHistory(repo ConfigHistoryRepo) ConfigHistory { + return ConfigHistory{ + repo: repo, + } } -func (c ConfigHistory) DeleteConfigVersion(ctx context.Context, id string) error { +func (s ConfigHistory) GetAllVersions(ctx context.Context, configId string) ([]domain.ConfigVersion, error) { + versions, err := s.repo.GetByConfigId(ctx, configId) + if err != nil { + return nil, errors.WithMessage(err, "get config versions") + } + + result := make([]domain.ConfigVersion, 0, len(versions)) + for _, version := range versions { + result = append(result, domain.ConfigVersion{ + Id: version.Id, + ConfigId: version.ConfigId, + ConfigVersion: version.Version, + Data: version.Data, + CreatedAt: time.Time(version.CreatedAt), + }) + } + + return result, nil +} +func (s ConfigHistory) Delete(ctx context.Context, id string) error { + err := s.repo.Delete(ctx, id) + if err != nil { + return errors.WithMessage(err, "delete config version") + } + return nil } diff --git a/service/api/config_schema.go b/service/api/config_schema.go index 7e6873f..1365cee 100644 --- a/service/api/config_schema.go +++ b/service/api/config_schema.go @@ -2,13 +2,40 @@ package api import ( "context" + "time" + "github.com/pkg/errors" "isp-config-service/domain" + "isp-config-service/entity" ) type ConfigSchema struct { + schemaRepo SchemaRepo } -func (c ConfigSchema) SchemaByModuleId(ctx context.Context, moduleId string) (*domain.ConfigSchema, error) { +func NewConfigSchema(schemaRepo SchemaRepo) ConfigSchema { + return ConfigSchema{ + schemaRepo: schemaRepo, + } +} + +func (s ConfigSchema) SchemaByModuleId(ctx context.Context, moduleId string) (*domain.ConfigSchema, error) { + schema, err := s.schemaRepo.GetByModuleId(ctx, moduleId) + if err != nil { + return nil, errors.WithMessage(err, "get config schema") + } + if schema == nil { + return nil, entity.ErrSchemaNotFound + } + + result := domain.ConfigSchema{ + Id: schema.Id, + Version: schema.ModuleVersion, + ModuleId: schema.ModuleId, + Schema: schema.Data, + CreatedAt: time.Time(schema.CreatedAt), + UpdatedAt: time.Time(schema.UpdatedAt), + } + return &result, nil } diff --git a/service/api/module.go b/service/api/module.go index c2ac3c8..f9ae576 100644 --- a/service/api/module.go +++ b/service/api/module.go @@ -14,7 +14,8 @@ import ( type ModuleRepo interface { All(ctx context.Context) ([]entity.Module, error) - Delete(ctx context.Context, idList []string) error + Delete(ctx context.Context, id string) error + GetByNames(ctx context.Context, names []string) ([]entity.Module, error) } type BackendsRepo interface { @@ -23,6 +24,7 @@ type BackendsRepo interface { type SchemaRepo interface { All(ctx context.Context) ([]entity.ConfigSchema, error) + GetByModuleId(ctx context.Context, moduleId string) (*entity.ConfigSchema, error) } type Module struct { @@ -107,8 +109,8 @@ func (s Module) Status(ctx context.Context) ([]domain.ModuleInfo, error) { return moduleInfos, nil } -func (s Module) Delete(ctx context.Context, idList []string) error { - err := s.moduleRepo.Delete(ctx, idList) +func (s Module) Delete(ctx context.Context, id string) error { + err := s.moduleRepo.Delete(ctx, id) if err != nil { return errors.WithMessage(err, "delete modules") } diff --git a/service/module/service.go b/service/module/service.go index 55a40db..84a6e6d 100644 --- a/service/module/service.go +++ b/service/module/service.go @@ -227,7 +227,7 @@ func (s Service) OnModuleConfigSchema( } config, err := s.configRepo.GetActive(ctx, moduleId) - if errors.Is(err, entity.ErrNoActiveConfig) { + if config == nil { md5Sum := md5.Sum([]byte(moduleId)) initialConfigId := hex.EncodeToString(md5Sum[:]) initialConfig := entity.Config{ diff --git a/service/subscription/service.go b/service/subscription/service.go index feea8ad..08a8b60 100644 --- a/service/subscription/service.go +++ b/service/subscription/service.go @@ -118,6 +118,9 @@ func (s Service) notifyBackendsChanged( if err != nil { return errors.WithMessage(err, "get module by id") } + if module == nil { + return errors.Errorf("unknown module: %s", moduleId) + } backends, err := s.backendRepo.GetByModuleId(ctx, moduleId) if err != nil { return errors.WithMessage(err, "get backends by module id") From cddd3383e879d2e257500876825d6a9da798c232 Mon Sep 17 00:00:00 2001 From: d1slike Date: Sun, 12 May 2024 16:35:33 +0300 Subject: [PATCH 09/40] Implement api --- assembly/locator.go | 18 +++- controller/api/config.go | 35 +++++-- controller/api/config_history.go | 17 +++- controller/api/config_schema.go | 19 +++- domain/errors.go | 30 +++++- entity/errors.go | 1 + repository/config.go | 47 +++++++++ repository/config_history.go | 18 ++++ service/api/config.go | 170 +++++++++++++++++++++++++++++-- service/api/config_history.go | 1 + 10 files changed, 333 insertions(+), 23 deletions(-) diff --git a/assembly/locator.go b/assembly/locator.go index b8e3cae..d7b9b9b 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -53,6 +53,7 @@ func (l Locator) Config() Config { eventRepo := repository.NewEvent(l.db) configRepo := repository.NewConfig(l.db) configSchemaRepo := repository.NewConfigSchema(l.db) + configHistoryRepo := repository.NewConfigHistory(l.db) etpSrv := etp.NewServer( etp.WithServerReadLimit(wsReadLimit), @@ -82,9 +83,20 @@ func (l Locator) Config() Config { moduleApiService := apisvs.NewModule(moduleRepo, backendRepo, configSchemaRepo) moduleApiController := api.NewModule(moduleApiService) - configApiController := api.NewConfig(nil) - configHistoryController := api.NewConfigHistory(nil) - configSchemaController := api.NewConfigSchema(nil) + configApiService := apisvs.NewConfig( + configRepo, + moduleRepo, + configSchemaRepo, + eventRepo, + configHistoryRepo, + ) + configApiController := api.NewConfig(configApiService) + + configHistoryApiService := apisvs.NewConfigHistory(configHistoryRepo) + configHistoryController := api.NewConfigHistory(configHistoryApiService) + + configSchemaApiService := apisvs.NewConfigSchema(configSchemaRepo) + configSchemaController := api.NewConfigSchema(configSchemaApiService) controllers := routes.Controllers{ Module: moduleController, diff --git a/controller/api/config.go b/controller/api/config.go index f38d37e..3419cd5 100644 --- a/controller/api/config.go +++ b/controller/api/config.go @@ -3,16 +3,19 @@ package api import ( "context" + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/grpc/apierrors" "isp-config-service/domain" + "isp-config-service/entity" ) type ConfigService interface { GetActiveConfigByModuleName(ctx context.Context, moduleName string) (*domain.Config, error) GetConfigsByModuleId(ctx context.Context, moduleId string) ([]domain.Config, error) - CreateUpdateConfig(ctx context.Context, moduleId string) (*domain.Config, error) + CreateUpdateConfig(ctx context.Context, adminId int, req domain.CreateUpdateConfigRequest) (*domain.Config, error) GetConfigById(ctx context.Context, configId string) (*domain.Config, error) MarkConfigAsActive(ctx context.Context, configId string) error - DeleteConfigs(ctx context.Context, idList []string) error + DeleteConfig(ctx context.Context, id string) error } type Config struct { @@ -33,11 +36,29 @@ func NewConfig(service ConfigService) Config { // @Produce json // @Param body body domain.GetByModuleNameRequest true "название модуля" // @Success 200 {object} domain.Config -// @Failure 404 {object} apierrors.Error "если конфигурация не найдена" +// @Failure 400 {object} apierrors.Error "если конфигурация не найдена" // @Failure 500 {object} apierrors.Error // @Router /config/get_active_config_by_module_name [POST] -func (c Config) GetActiveConfigByModuleName(ctx context.Context, request domain.GetByModuleNameRequest) (*domain.Config, error) { - +func (c Config) GetActiveConfigByModuleName(ctx context.Context, req domain.GetByModuleNameRequest) (*domain.Config, error) { + config, err := c.service.GetActiveConfigByModuleName(ctx, req.ModuleName) + switch { + case errors.Is(err, entity.ErrModuleNotFound): + return nil, apierrors.NewBusinessError( + domain.ErrorCodeModuleNotFound, + "module not found", + err, + ) + case errors.Is(err, entity.ErrConfigNotFound): + return nil, apierrors.NewBusinessError( + domain.ErrorCodeModuleNotFound, + "active config not found", + err, + ) + case err != nil: + return nil, apierrors.NewInternalServiceError(err) + default: + return config, nil + } } // GetConfigsByModuleId @@ -52,8 +73,8 @@ func (c Config) GetActiveConfigByModuleName(ctx context.Context, request domain. // @Failure 404 {object} apierrors.Error "если конфигурация не найдена" // @Failure 500 {object} apierrors.Error // @Router /config/get_configs_by_module_id [POST] -func (c Config) GetConfigsByModuleId(ctx context.Context, request domain.GetByModuleIdRequest) ([]domain.Config, error) { - +func (c Config) GetConfigsByModuleId(ctx context.Context, req domain.GetByModuleIdRequest) ([]domain.Config, error) { + c.service.GetConfigsByModuleId(, re) } // CreateUpdateConfig diff --git a/controller/api/config_history.go b/controller/api/config_history.go index dcf535a..844bccd 100644 --- a/controller/api/config_history.go +++ b/controller/api/config_history.go @@ -3,6 +3,7 @@ package api import ( "context" + "github.com/txix-open/isp-kit/grpc/apierrors" "isp-config-service/domain" ) @@ -33,7 +34,11 @@ func NewConfigHistory(service ConfigHistoryService) ConfigHistory { // @Failure 500 {object} apierrors.Error // @Router /config/get_all_version [POST] func (c ConfigHistory) GetAllVersion(ctx context.Context, req domain.ConfigIdRequest) ([]domain.ConfigVersion, error) { - + versions, err := c.service.GetAllVersions(ctx, req.Id) + if err != nil { + return nil, apierrors.NewInternalServiceError(err) + } + return versions, nil } // DeleteConfigVersion @@ -44,9 +49,15 @@ func (c ConfigHistory) GetAllVersion(ctx context.Context, req domain.ConfigIdReq // @Produce json // @Param body body domain.ConfigIdRequest true "id версии конфигурации" // @Success 200 {object} domain.DeleteResponse -// @Failure 400 {object} apierrors.Error "если не указан массив идентификаторов" +// @Failure 400 {object} apierrors.Error "не указан массив идентификаторов" // @Failure 500 {object} apierrors.Error // @Router /config/delete_version [POST] func (c ConfigHistory) DeleteConfigVersion(ctx context.Context, req domain.ConfigIdRequest) (*domain.DeleteResponse, error) { - + err := c.service.Delete(ctx, req.Id) + if err != nil { + return nil, apierrors.NewInternalServiceError(err) + } + return &domain.DeleteResponse{ + Deleted: 1, + }, nil } diff --git a/controller/api/config_schema.go b/controller/api/config_schema.go index 94c4b70..6efde60 100644 --- a/controller/api/config_schema.go +++ b/controller/api/config_schema.go @@ -3,7 +3,10 @@ package api import ( "context" + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/grpc/apierrors" "isp-config-service/domain" + "isp-config-service/entity" ) type ConfigSchemaService interface { @@ -28,9 +31,21 @@ func NewConfigSchema(service ConfigSchemaService) ConfigSchema { // @Produce json // @Param body body domain.GetByModuleIdRequest true "идентификатор модуля" // @Success 200 {object} domain.ConfigSchema -// @Failure 404 {object} apierrors.Error "если схема для модуля не найдена" +// @Failure 400 {object} apierrors.Error "`errorCode: 2005` - схема для модуля не найдена" // @Failure 500 {object} apierrors.Error // @Router /schema/get_by_module_id [POST] func (c ConfigSchema) SchemaByModuleId(ctx context.Context, request domain.GetByModuleIdRequest) (*domain.ConfigSchema, error) { - + schema, err := c.service.SchemaByModuleId(ctx, request.ModuleId) + switch { + case errors.Is(err, entity.ErrSchemaNotFound): + return nil, apierrors.NewBusinessError( + domain.ErrorCodeSchemaNotFound, + "config schema not found", + err, + ) + case err != nil: + return nil, apierrors.NewInternalServiceError(err) + default: + return schema, nil + } } diff --git a/domain/errors.go b/domain/errors.go index ffe1036..ab2e4ae 100644 --- a/domain/errors.go +++ b/domain/errors.go @@ -1,5 +1,33 @@ package domain +import ( + "fmt" + "strings" +) + const ( - ErrorCodeBadRequest = 400 + ErrorCodeBadRequest = 400 + ErrorCodeModuleNotFound = 2001 + ErrorCodeConfigNotFound = 2002 + ErrorCodeInvalidConfig = 2003 + ErrorCodeConfigVersionConflict = 2004 + ErrorCodeSchemaNotFound = 2005 ) + +type ConfigValidationError struct { + Details map[string]string +} + +func NewConfigValidationError(details map[string]string) ConfigValidationError { + return ConfigValidationError{ + Details: details, + } +} + +func (e ConfigValidationError) Error() string { + descriptions := make([]string, 0, len(e.Details)) + for field, err := range e.Details { + descriptions = append(descriptions, fmt.Sprintf("%s -> %s", field, err)) + } + return strings.Join(descriptions, "; ") +} diff --git a/entity/errors.go b/entity/errors.go index 02b6c98..8e14039 100644 --- a/entity/errors.go +++ b/entity/errors.go @@ -9,4 +9,5 @@ var ( ErrModuleNotFound = errors.New("module not found") ErrSchemaNotFound = errors.New("config schema not found") ErrConfigNotFoundOrActive = errors.New("config not found or markers as active") + ErrConfigConflictUpdate = errors.New("config conflict update") ) diff --git a/repository/config.go b/repository/config.go index b0ffe27..e26cb6f 100644 --- a/repository/config.go +++ b/repository/config.go @@ -6,6 +6,7 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" "isp-config-service/entity" + "isp-config-service/entity/xtypes" "isp-config-service/service/rqlite/db" ) @@ -107,3 +108,49 @@ func (r Config) DeleteNonActiveById(ctx context.Context, id string) (bool, error } return affected > 0, nil } + +func (r Config) UpdateByVersion(ctx context.Context, cfg entity.Config) (bool, error) { + query, args, err := squirrel.Update(Table("config")). + SetMap(map[string]interface{}{ + "name": cfg.Name, + "data": cfg.Data, + "version": squirrel.Expr("version + 1"), + "updated_at": squirrel.Expr("unixepoch()"), + }).Where(squirrel.Eq{ + "id": cfg.Id, + "version": cfg.Version, + }).ToSql() + if err != nil { + return false, errors.WithMessage(err, "build query") + } + + result, err := r.db.Exec(ctx, query, args...) + if err != nil { + return false, errors.WithMessagef(err, "exec: %s", query) + } + + affected, err := result.RowsAffected() + if err != nil { + return false, errors.WithMessage(err, "get rows affected") + } + return affected > 0, nil +} + +func (r Config) SetActive(ctx context.Context, configId string, active xtypes.Bool) error { + query, args, err := squirrel.Update(Table("config")). + Set("active", active). + Set("updated_at", squirrel.Expr("unixepoch()")). + Where(squirrel.Eq{ + "id": configId, + }).ToSql() + if err != nil { + return errors.WithMessage(err, "build query") + } + + _, err = r.db.Exec(ctx, query, args...) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + + return nil +} diff --git a/repository/config_history.go b/repository/config_history.go index d4b90f0..47f6cfb 100644 --- a/repository/config_history.go +++ b/repository/config_history.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/Masterminds/squirrel" "github.com/pkg/errors" "isp-config-service/entity" "isp-config-service/service/rqlite/db" @@ -37,3 +38,20 @@ func (r ConfigHistory) Delete(ctx context.Context, id string) error { } return nil } + +func (r ConfigHistory) Insert(ctx context.Context, history entity.ConfigHistory) error { + query, args, err := squirrel.Insert(Table("config_history")). + Columns("id", "config_id", "data", "version", "admin_id"). + Values(history.Id, history.ConfigId, history.Data, history.Version, history.AdminId). + ToSql() + if err != nil { + return errors.WithMessage(err, "build query") + } + + _, err = r.db.Exec(ctx, query, args...) + if err != nil { + return errors.WithMessagef(err, "exec: %s", query) + } + + return nil +} diff --git a/service/api/config.go b/service/api/config.go index 7e9c0ac..e14fbda 100644 --- a/service/api/config.go +++ b/service/api/config.go @@ -4,10 +4,12 @@ import ( "context" "time" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/xeipuuv/gojsonschema" "isp-config-service/domain" "isp-config-service/entity" + "isp-config-service/entity/xtypes" ) type ConfigRepo interface { @@ -15,23 +17,36 @@ type ConfigRepo interface { GetByModuleId(ctx context.Context, moduleId string) ([]entity.Config, error) GetById(ctx context.Context, id string) (*entity.Config, error) DeleteNonActiveById(ctx context.Context, id string) (bool, error) + Insert(ctx context.Context, cfg entity.Config) error + UpdateByVersion(ctx context.Context, cfg entity.Config) (bool, error) + SetActive(ctx context.Context, configId string, active xtypes.Bool) error +} + +type EventRepo interface { + Insert(ctx context.Context, event entity.Event) error } type Config struct { - configRepo ConfigRepo - moduleRepo ModuleRepo - schemaRepo SchemaRepo + configRepo ConfigRepo + moduleRepo ModuleRepo + schemaRepo SchemaRepo + eventRepo EventRepo + configHistoryRepo ConfigHistoryRepo } func NewConfig( configRepo ConfigRepo, moduleRepo ModuleRepo, schemaRepo SchemaRepo, + eventRepo EventRepo, + configHistoryRepo ConfigHistoryRepo, ) Config { return Config{ - configRepo: configRepo, - moduleRepo: moduleRepo, - schemaRepo: schemaRepo, + configRepo: configRepo, + moduleRepo: moduleRepo, + schemaRepo: schemaRepo, + eventRepo: eventRepo, + configHistoryRepo: configHistoryRepo, } } @@ -83,8 +98,86 @@ func (c Config) GetConfigsByModuleId(ctx context.Context, moduleId string) ([]do return result, nil } -func (c Config) CreateUpdateConfig(ctx context.Context, moduleId string) (*domain.Config, error) { +func (c Config) CreateUpdateConfig( + ctx context.Context, + adminId int, + req domain.CreateUpdateConfigRequest, +) (*domain.Config, error) { + if !req.Unsafe { + err := c.validateConfigUpdate(ctx, req.ModuleId, req.Data) + if err != nil { + return nil, errors.WithMessage(err, "validate config") + } + } + + if req.Id == "" { + config := entity.Config{ + Id: uuid.NewString(), + Name: req.Name, + ModuleId: req.ModuleId, + Data: req.Data, + Version: 1, + Active: false, + } + err := c.configRepo.Insert(ctx, config) + if err != nil { + return nil, errors.WithMessage(err, "insert new config") + } + result := configToDto(config, nil) + return &result, nil + } + + oldConfig, err := c.configRepo.GetById(ctx, req.Id) + if err != nil { + return nil, errors.WithMessage(err, "get old config") + } + if oldConfig == nil { + return nil, entity.ErrConfigNotFound + } + + config := entity.Config{ + Id: req.Id, + Name: req.Name, + Data: req.Data, + Version: req.Version, + } + updated, err := c.configRepo.UpdateByVersion(ctx, config) + if err != nil { + return nil, errors.WithMessage(err, "update config") + } + if !updated { + return nil, entity.ErrConfigConflictUpdate + } + + history := entity.ConfigHistory{ + Id: uuid.NewString(), + ConfigId: req.Id, + Data: oldConfig.Data, + Version: oldConfig.Version, + AdminId: adminId, + } + err = c.configHistoryRepo.Insert(ctx, history) + if err != nil { + return nil, errors.WithMessage(err, "save config history") + } + + if oldConfig.Active { + err = c.emitChangeActiveConfigEvent(ctx, oldConfig.ModuleId) + if err != nil { + return nil, errors.WithMessage(err, "emit change active config event") + } + } + newConfig := entity.Config{ + Id: req.Id, + Name: req.Name, + ModuleId: req.ModuleId, + Data: req.Data, + Version: req.Version + 1, + Active: oldConfig.Active, + } + result := configToDto(newConfig, nil) + return &result, nil } func (c Config) GetConfigById(ctx context.Context, configId string) (*domain.Config, error) { @@ -101,7 +194,36 @@ func (c Config) GetConfigById(ctx context.Context, configId string) (*domain.Con } func (c Config) MarkConfigAsActive(ctx context.Context, configId string) error { + cfg, err := c.configRepo.GetById(ctx, configId) + if err != nil { + return errors.WithMessage(err, "get config by id") + } + if cfg == nil { + return entity.ErrConfigNotFound + } + currentActiveConfig, err := c.configRepo.GetActive(ctx, cfg.ModuleId) + if err != nil { + return errors.WithMessage(err, "get active config") + } + if currentActiveConfig != nil { + err := c.configRepo.SetActive(ctx, currentActiveConfig.Id, false) + if err != nil { + return errors.WithMessage(err, "deactivate config") + } + } + + err = c.configRepo.SetActive(ctx, configId, true) + if err != nil { + return errors.WithMessage(err, "set active config") + } + + err = c.emitChangeActiveConfigEvent(ctx, cfg.ModuleId) + if err != nil { + return errors.WithMessage(err, "emit change active config event") + } + + return nil } func (c Config) DeleteConfig(ctx context.Context, configId string) error { @@ -115,6 +237,40 @@ func (c Config) DeleteConfig(ctx context.Context, configId string) error { return nil } +func (c Config) validateConfigUpdate(ctx context.Context, moduleId string, configData []byte) error { + schema, err := c.schemaRepo.GetByModuleId(ctx, moduleId) + if err != nil { + return errors.WithMessage(err, "get schema by module id") + } + if schema == nil { + return entity.ErrSchemaNotFound + } + + details, err := validateConfig(configData, schema.Data) + if err != nil { + return errors.WithMessage(err, "validate config") + } + if len(details) == 0 { + return nil + } + + return domain.NewConfigValidationError(details) +} + +func (c Config) emitChangeActiveConfigEvent(ctx context.Context, moduleId string) error { + eventPayload := entity.EventPayload{ + ConfigUpdated: &entity.PayloadConfigUpdated{ + ModuleId: moduleId, + }, + } + err := c.eventRepo.Insert(ctx, entity.NewEvent(eventPayload)) + if err != nil { + return errors.WithMessage(err, "insert new event") + } + + return nil +} + func configToDto(config entity.Config, schema []byte) domain.Config { valid := true if schema != nil { diff --git a/service/api/config_history.go b/service/api/config_history.go index a0d9816..701084c 100644 --- a/service/api/config_history.go +++ b/service/api/config_history.go @@ -12,6 +12,7 @@ import ( type ConfigHistoryRepo interface { Delete(ctx context.Context, id string) error GetByConfigId(ctx context.Context, configId string) ([]entity.ConfigHistory, error) + Insert(ctx context.Context, history entity.ConfigHistory) error } type ConfigHistory struct { From 3b34468842808f53e425f00e33d98a8c9b77f25a Mon Sep 17 00:00:00 2001 From: d1slike Date: Tue, 14 May 2024 14:34:13 +0300 Subject: [PATCH 10/40] Improve logging --- assembly/locator.go | 5 +- controller/api/config.go | 126 +++++++- controller/module.go | 4 +- docs/swagger.yaml | 499 +++++++++++++++++++++++++++++ domain/request.go | 2 +- domain/response.go | 8 +- entity/config.go | 1 + go.mod | 9 +- go.sum | 7 + helpers/helpers.go | 9 + main.go | 2 +- middlewares/sql.go | 32 ++ middlewares/sql_metrics/context.go | 20 ++ middlewares/ws_log.go | 32 ++ migrations/20240503120009_init.sql | 5 +- repository/backend.go | 9 + repository/config.go | 24 +- repository/config_history.go | 7 + repository/config_schema.go | 7 + repository/event.go | 7 + repository/module.go | 13 + routes/routes.go | 19 +- service/api/config.go | 4 +- service/module/emitter.go | 59 ++++ service/module/service.go | 32 +- service/startup/service.go | 7 +- service/subscription/service.go | 37 +-- 27 files changed, 906 insertions(+), 80 deletions(-) create mode 100644 docs/swagger.yaml create mode 100644 middlewares/sql.go create mode 100644 middlewares/sql_metrics/context.go create mode 100644 middlewares/ws_log.go create mode 100644 service/module/emitter.go diff --git a/assembly/locator.go b/assembly/locator.go index d7b9b9b..c606933 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -61,11 +61,13 @@ func (l Locator) Config() Config { InsecureSkipVerify: true, }), ) + emitter := module.NewEmitter(l.logger) subscriptionService := subscription.NewService( moduleRepo, backendRepo, configRepo, etpSrv.Rooms(), + emitter, l.logger, ) @@ -76,6 +78,7 @@ func (l Locator) Config() Config { configRepo, configSchemaRepo, subscriptionService, + emitter, l.logger, ) moduleController := controller.NewModule(moduleService, l.logger) @@ -108,7 +111,7 @@ func (l Locator) Config() Config { mapper := endpoint.DefaultWrapper(l.logger) grpcMux := routes.GrpcHandler(mapper, controllers) - routes.BindEtp(etpSrv, controllers) + routes.BindEtp(etpSrv, controllers, l.logger) httpMux := routes.HttpHandler(etpSrv) diff --git a/controller/api/config.go b/controller/api/config.go index 3419cd5..8c40e56 100644 --- a/controller/api/config.go +++ b/controller/api/config.go @@ -2,9 +2,12 @@ package api import ( "context" + "strconv" "github.com/pkg/errors" + "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/grpc/apierrors" + "google.golang.org/grpc/metadata" "isp-config-service/domain" "isp-config-service/entity" ) @@ -36,7 +39,7 @@ func NewConfig(service ConfigService) Config { // @Produce json // @Param body body domain.GetByModuleNameRequest true "название модуля" // @Success 200 {object} domain.Config -// @Failure 400 {object} apierrors.Error "если конфигурация не найдена" +// @Failure 400 {object} apierrors.Error "`errorCode: 2001` - модуль не найден
`errorCode: 2002` - конфиг не найден" // @Failure 500 {object} apierrors.Error // @Router /config/get_active_config_by_module_name [POST] func (c Config) GetActiveConfigByModuleName(ctx context.Context, req domain.GetByModuleNameRequest) (*domain.Config, error) { @@ -50,7 +53,7 @@ func (c Config) GetActiveConfigByModuleName(ctx context.Context, req domain.GetB ) case errors.Is(err, entity.ErrConfigNotFound): return nil, apierrors.NewBusinessError( - domain.ErrorCodeModuleNotFound, + domain.ErrorCodeConfigNotFound, "active config not found", err, ) @@ -70,11 +73,16 @@ func (c Config) GetActiveConfigByModuleName(ctx context.Context, req domain.GetB // @Param body body domain.GetByModuleIdRequest true "ID модуля" // @Success 200 {array} domain.Config // @Failure 400 {object} apierrors.Error "если идентификатор не указан" -// @Failure 404 {object} apierrors.Error "если конфигурация не найдена" // @Failure 500 {object} apierrors.Error // @Router /config/get_configs_by_module_id [POST] func (c Config) GetConfigsByModuleId(ctx context.Context, req domain.GetByModuleIdRequest) ([]domain.Config, error) { - c.service.GetConfigsByModuleId(, re) + configs, err := c.service.GetConfigsByModuleId(ctx, req.ModuleId) + switch { + case err != nil: + return nil, apierrors.NewInternalServiceError(err) + default: + return configs, nil + } } // CreateUpdateConfig @@ -86,26 +94,83 @@ func (c Config) GetConfigsByModuleId(ctx context.Context, req domain.GetByModule // @Produce json // @Param body body domain.CreateUpdateConfigRequest true "объект для сохранения" // @Success 200 {object} domain.Config -// @Failure 404 {object} apierrors.Error "если конфигурация не найдена" +// @Failure 400 {object} apierrors.Error "`errorCode: 2003` - конфиг не соотвествует текущей схеме
`errorCode: 2002` - указанного id не сущесвует
`errorCode: 2004` - кто-то уже обновил конфигурацию
`errorCode: 2005` - схема конфигурации не найдена
" // @Failure 500 {object} apierrors.Error // @Router /config/create_update_config [POST] -func (c Config) CreateUpdateConfig(ctx context.Context, config domain.CreateUpdateConfigRequest) (*domain.Config, error) { +func (c Config) CreateUpdateConfig( + ctx context.Context, + authData grpc.AuthData, + req domain.CreateUpdateConfigRequest, +) (*domain.Config, error) { + adminIdValue, _ := grpc.StringFromMd("x-admin-id", metadata.MD(authData)) + var adminId int + if adminIdValue != "" { + adminId, _ = strconv.Atoi(adminIdValue) + } + config, err := c.service.CreateUpdateConfig(ctx, adminId, req) + + var validationError domain.ConfigValidationError + switch { + case errors.As(err, &validationError): + details := map[string]any{} + for key, value := range validationError.Details { + details[key] = value + } + return nil, apierrors.NewBusinessError( + domain.ErrorCodeInvalidConfig, + "invalid config", + err, + ).WithDetails(details) + case errors.Is(err, entity.ErrConfigNotFound): + return nil, apierrors.NewBusinessError( + domain.ErrorCodeConfigNotFound, + "config not found", + err, + ) + case errors.Is(err, entity.ErrConfigConflictUpdate): + return nil, apierrors.NewBusinessError( + domain.ErrorCodeConfigVersionConflict, + "someone has updated the config", + err, + ) + case errors.Is(err, entity.ErrSchemaNotFound): + return nil, apierrors.NewBusinessError( + domain.ErrorCodeSchemaNotFound, + "config schema not found", + err, + ) + case err != nil: + return nil, apierrors.NewInternalServiceError(err) + default: + return config, nil + } } // GetConfigById -// @Summary Метод получение актуальной конфигурации конфигурации -// @Description Возвращает актуальную версию конфигурации без дополнительного содержимого (ConfigData) +// @Summary Метод получение конфигурации по id // @Tags Конфигурация // @Accept json // @Produce json // @Param body body domain.ConfigIdRequest true "id конфигурации" // @Success 200 {object} domain.Config -// @Failure 400 {object} apierrors.Error "если не указан идентификатор конфигурации" +// @Failure 400 {object} apierrors.Error "`errorCode: 2002` - конфиг не найден
" // @Failure 500 {object} apierrors.Error // @Router /config/get_config_by_id [POST] -func (c Config) GetConfigById(ctx context.Context, req domain.ConfigIdRequest) (domain.Config, error) { - +func (c Config) GetConfigById(ctx context.Context, req domain.ConfigIdRequest) (*domain.Config, error) { + config, err := c.service.GetConfigById(ctx, req.Id) + switch { + case errors.Is(err, entity.ErrConfigNotFound): + return nil, apierrors.NewBusinessError( + domain.ErrorCodeConfigNotFound, + "config not found", + err, + ) + case err != nil: + return nil, apierrors.NewInternalServiceError(err) + default: + return config, nil + } } // MarkConfigAsActive @@ -115,12 +180,24 @@ func (c Config) GetConfigById(ctx context.Context, req domain.ConfigIdRequest) ( // @Accept json // @Produce json // @Param body body domain.ConfigIdRequest true "id конфигурации для изменения" -// @Success 200 {object} domain.Config "активированная конфигурация" -// @Failure 404 {object} apierrors.Error "если конфигурация не найдена" +// @Success 200 +// @Failure 400 {object} apierrors.Error "`errorCode: 2002` - конфиг не найден
" // @Failure 500 {object} apierrors.Error // @Router /config/mark_config_as_active [POST] -func (c Config) MarkConfigAsActive(ctx context.Context, identity domain.ConfigIdRequest) (*domain.Config, error) { - +func (c Config) MarkConfigAsActive(ctx context.Context, identity domain.ConfigIdRequest) error { + err := c.service.MarkConfigAsActive(ctx, identity.Id) + switch { + case errors.Is(err, entity.ErrConfigNotFound): + return apierrors.NewBusinessError( + domain.ErrorCodeConfigNotFound, + "config not found", + err, + ) + case err != nil: + return apierrors.NewInternalServiceError(err) + default: + return nil + } } // DeleteConfigs @@ -135,5 +212,24 @@ func (c Config) MarkConfigAsActive(ctx context.Context, identity domain.ConfigId // @Failure 500 {object} apierrors.Error // @Router /config/delete_config [POST] func (c Config) DeleteConfigs(ctx context.Context, identities []string) (*domain.DeleteResponse, error) { + configId, err := getSingleId(identities) + if err != nil { + return nil, err + } + err = c.service.DeleteConfig(ctx, configId) + switch { + case errors.Is(err, entity.ErrConfigNotFoundOrActive): + return nil, apierrors.NewBusinessError( + domain.ErrorCodeConfigNotFound, + "config not found or is active", + err, + ) + case err != nil: + return nil, apierrors.NewInternalServiceError(err) + default: + return &domain.DeleteResponse{ + Deleted: 1, + }, nil + } } diff --git a/controller/module.go b/controller/module.go index 8ea1f95..2123515 100644 --- a/controller/module.go +++ b/controller/module.go @@ -19,7 +19,7 @@ var ( type ModuleService interface { OnConnect(ctx context.Context, conn *etp.Conn, moduleName string) error OnDisconnect(ctx context.Context, conn *etp.Conn, moduleName string, isNormalClose bool, err error) error - OnError(ctx context.Context, conn *etp.Conn, moduleName string, err error) + OnError(ctx context.Context, conn *etp.Conn, err error) OnModuleReady( ctx context.Context, conn *etp.Conn, @@ -86,7 +86,7 @@ func (m Module) OnDisconnect(conn *etp.Conn, err error) { } func (m Module) OnError(conn *etp.Conn, err error) { - m.service.OnError(conn.HttpRequest().Context(), conn, helpers.ModuleName(conn), err) + m.service.OnError(conn.HttpRequest().Context(), conn, err) } func (m Module) OnModuleReady(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { diff --git a/docs/swagger.yaml b/docs/swagger.yaml new file mode 100644 index 0000000..6ccf683 --- /dev/null +++ b/docs/swagger.yaml @@ -0,0 +1,499 @@ +basePath: /api/config +definitions: + cluster.AddressConfiguration: + properties: + ip: + type: string + port: + type: string + type: object + cluster.EndpointDescriptor: + properties: + extra: + additionalProperties: {} + type: object + inner: + type: boolean + path: + type: string + userAuthRequired: + type: boolean + type: object + domain.Config: + properties: + active: + type: boolean + createdAt: + type: string + data: + type: object + id: + type: string + moduleId: + type: string + name: + type: string + updatedAt: + type: string + valid: + type: boolean + version: + type: integer + type: object + domain.ConfigIdRequest: + properties: + id: + type: string + required: + - id + type: object + domain.ConfigSchema: + properties: + createdAt: + type: string + id: + type: string + moduleId: + type: string + schema: + type: object + updatedAt: + type: string + version: + type: string + type: object + domain.ConfigVersion: + properties: + configId: + type: string + configVersion: + type: integer + createdAt: + type: string + data: + type: object + id: + type: string + type: object + domain.Connection: + properties: + address: + $ref: '#/definitions/cluster.AddressConfiguration' + endpoints: + items: + $ref: '#/definitions/cluster.EndpointDescriptor' + type: array + establishedAt: + type: string + libVersion: + type: string + version: + type: string + type: object + domain.CreateUpdateConfigRequest: + properties: + data: + type: object + id: + type: string + moduleId: + type: string + name: + type: string + unsafe: + type: boolean + version: + type: integer + required: + - moduleId + - name + type: object + domain.DeleteResponse: + properties: + deleted: + type: integer + type: object + domain.GetByModuleIdRequest: + properties: + moduleId: + type: string + required: + - moduleId + type: object + domain.GetByModuleNameRequest: + properties: + moduleName: + type: string + required: + - moduleName + type: object + domain.ModuleInfo: + properties: + active: + type: boolean + configSchema: + type: object + createdAt: + type: string + id: + type: string + lastConnectedAt: + type: string + lastDisconnectedAt: + type: string + name: + type: string + status: + items: + $ref: '#/definitions/domain.Connection' + type: array + type: object + github_com_txix-open_isp-kit_grpc_apierrors.Error: + properties: + details: + additionalProperties: true + type: object + errorCode: + type: integer + errorMessage: + type: string + type: object +host: localhost:9000 +info: + contact: {} + description: Модуль управления конфигурациями + license: + name: GNU GPL v3.0 + title: isp-config-service + version: 1.0.0 +paths: + /config/create_update_config: + post: + consumes: + - application/json + description: |- + Если конфиг с таким id существует, то обновляет данные, если нет, то добавляет данные в базу + В случае обновления рассылает всем подключенным модулям актуальную конфигурацию + parameters: + - description: объект для сохранения + in: body + name: body + required: true + schema: + $ref: '#/definitions/domain.CreateUpdateConfigRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/domain.Config' + "400": + description: '`errorCode: 2003` - конфиг не соотвествует текущей схеме
`errorCode: + 2002` - указанного id не сущесвует
`errorCode: 2004` - кто-то уже + обновил конфигурацию
`errorCode: 2005` - схема конфигурации не найдена
' + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод обновления конфигурации + tags: + - Конфигурация + /config/delete_config: + post: + consumes: + - application/json + description: Возвращает количество удаленных модулей + parameters: + - description: массив идентификаторов конфигураций + in: body + name: body + required: true + schema: + items: + type: string + type: array + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/domain.DeleteResponse' + "400": + description: если не указан массив идентификаторов + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод удаления объектов конфигурации по идентификаторам + tags: + - Конфигурация + /config/delete_version: + post: + consumes: + - application/json + description: Возвращает количество удаленных версий + parameters: + - description: id версии конфигурации + in: body + name: body + required: true + schema: + $ref: '#/definitions/domain.ConfigIdRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/domain.DeleteResponse' + "400": + description: не указан массив идентификаторов + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод удаления версии конфигурации + tags: + - Конфигурация + /config/get_active_config_by_module_name: + post: + consumes: + - application/json + description: Возвращает активную конфиграцию по названию модуля + parameters: + - description: название модуля + in: body + name: body + required: true + schema: + $ref: '#/definitions/domain.GetByModuleNameRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/domain.Config' + "400": + description: '`errorCode: 2001` - модуль не найден
`errorCode: 2002` + - конфиг не найден' + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод получения объекта конфигурации по названию модуля + tags: + - Конфигурация + /config/get_all_version: + post: + consumes: + - application/json + description: Возвращает предыдущие версии конфигураций + parameters: + - description: id конфигурации + in: body + name: body + required: true + schema: + $ref: '#/definitions/domain.ConfigIdRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/domain.ConfigVersion' + type: array + "400": + description: если не указан массив идентификаторов + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод получение старых версий конфигурации + tags: + - Конфигурация + /config/get_config_by_id: + post: + consumes: + - application/json + parameters: + - description: id конфигурации + in: body + name: body + required: true + schema: + $ref: '#/definitions/domain.ConfigIdRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/domain.Config' + "400": + description: '`errorCode: 2002` - конфиг не найден
' + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод получение конфигурации по id + tags: + - Конфигурация + /config/get_configs_by_module_id: + post: + consumes: + - application/json + description: Возвращает список конфиграции по ID модуля + parameters: + - description: ID модуля + in: body + name: body + required: true + schema: + $ref: '#/definitions/domain.GetByModuleIdRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/domain.Config' + type: array + "400": + description: если идентификатор не указан + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод получения списка конфигураций по ID модуля + tags: + - Конфигурация + /config/mark_config_as_active: + post: + consumes: + - application/json + description: Активирует указанную конфигурацию и деактивирует остальные, возвращает + активированную конфигурацию + parameters: + - description: id конфигурации для изменения + in: body + name: body + required: true + schema: + $ref: '#/definitions/domain.ConfigIdRequest' + produces: + - application/json + responses: + "200": + description: OK + "400": + description: '`errorCode: 2002` - конфиг не найден
' + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод активации конфигурации для модуля + tags: + - Конфигурация + /module/delete_module: + post: + consumes: + - application/json + description: Возвращает количество удаленных модулей + parameters: + - description: массив идентификаторов модулей + in: body + name: body + required: true + schema: + items: + type: string + type: array + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/domain.DeleteResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод удаления объектов модулей по идентификаторам + tags: + - Модули + /module/get_modules_info: + post: + consumes: + - application/json + description: Возвращает полное состояние всех модулей в кластере (схема конфигурации, + подключенные экземпляры) + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/domain.ModuleInfo' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод полчения полной информации о состоянии модулей + tags: + - Модули + /schema/get_by_module_id: + post: + consumes: + - application/json + description: Возвращает текущую json схему конфигурации модуля + parameters: + - description: идентификатор модуля + in: body + name: body + required: true + schema: + $ref: '#/definitions/domain.GetByModuleIdRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/domain.ConfigSchema' + "400": + description: '`errorCode: 2005` - схема для модуля не найдена' + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод получения схемы конфигурации модуля + tags: + - Схема +swagger: "2.0" diff --git a/domain/request.go b/domain/request.go index fc973cc..65cb20b 100644 --- a/domain/request.go +++ b/domain/request.go @@ -21,6 +21,6 @@ type CreateUpdateConfigRequest struct { Name string `validate:"required"` ModuleId string `validate:"required"` Version int - Data json.RawMessage + Data json.RawMessage `swaggertype:"object"` Unsafe bool } diff --git a/domain/response.go b/domain/response.go index 67b5b87..bc92cfe 100644 --- a/domain/response.go +++ b/domain/response.go @@ -17,7 +17,7 @@ type ModuleInfo struct { Active bool LastConnectedAt *time.Time LastDisconnectedAt *time.Time - ConfigSchema json.RawMessage + ConfigSchema json.RawMessage `swaggertype:"object"` Status []Connection CreatedAt time.Time } @@ -35,7 +35,7 @@ type Config struct { Name string ModuleId string Valid bool - Data json.RawMessage + Data json.RawMessage `swaggertype:"object"` Version int Active bool CreatedAt time.Time @@ -46,7 +46,7 @@ type ConfigVersion struct { Id string ConfigId string ConfigVersion int - Data json.RawMessage + Data json.RawMessage `swaggertype:"object"` CreatedAt time.Time } @@ -54,7 +54,7 @@ type ConfigSchema struct { Id string Version string ModuleId string - Schema json.RawMessage + Schema json.RawMessage `swaggertype:"object"` CreatedAt time.Time UpdatedAt time.Time } diff --git a/entity/config.go b/entity/config.go index 0382c46..ac3d8d4 100644 --- a/entity/config.go +++ b/entity/config.go @@ -11,6 +11,7 @@ type Config struct { Data []byte `json:"data"` Version int `json:"version"` Active xtypes.Bool `json:"active"` + AdminId int `json:"admin_id"` CreatedAt xtypes.Time `json:"created_at"` UpdatedAt xtypes.Time `json:"updated_at"` } diff --git a/go.mod b/go.mod index 24cdbb9..bde37b9 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,15 @@ require ( github.com/google/uuid v1.6.0 github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.20.0 + github.com/prometheus/client_golang v1.19.0 github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 github.com/rqlite/rqlite/v8 v8.23.4 github.com/tidwall/gjson v1.17.1 github.com/txix-open/etp/v3 v3.1.0 github.com/txix-open/isp-kit v1.31.2 + github.com/xeipuuv/gojsonschema v1.2.0 + golang.org/x/sync v0.7.0 + google.golang.org/grpc v1.63.2 ) require ( @@ -49,7 +53,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.19.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.14.0 // indirect @@ -63,6 +66,8 @@ require ( github.com/txix-open/jsonschema v1.2.0 // indirect github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.etcd.io/bbolt v1.3.9 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.51.0 // indirect go.opentelemetry.io/otel v1.26.0 // indirect @@ -76,12 +81,10 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/net v0.24.0 // indirect - golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index e6dc68b..5b96323 100644 --- a/go.sum +++ b/go.sum @@ -237,6 +237,13 @@ github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+ github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= diff --git a/helpers/helpers.go b/helpers/helpers.go index f4760de..ec30041 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -2,10 +2,12 @@ package helpers import ( "net" + "strconv" "github.com/pkg/errors" "github.com/txix-open/etp/v3" "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/log" "isp-config-service/entity" ) @@ -23,3 +25,10 @@ func SplitAddress(backend entity.Backend) (cluster.AddressConfiguration, error) Port: port, }, nil } + +func LogFields(conn *etp.Conn) []log.Field { + return []log.Field{ + log.String("moduleName", ModuleName(conn)), + log.String("connId", strconv.FormatUint(conn.Id(), 10)), + } +} diff --git a/main.go b/main.go index 83d1c7a..a04f632 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,7 @@ var ( ) // @title isp-config-service -// @version 1.0.0 +// @version 3.0.0 // @description Модуль управления конфигурациями // @license.name GNU GPL v3.0 diff --git a/middlewares/sql.go b/middlewares/sql.go new file mode 100644 index 0000000..5c836ce --- /dev/null +++ b/middlewares/sql.go @@ -0,0 +1,32 @@ +package middlewares + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/txix-open/isp-kit/http/httpcli" + "github.com/txix-open/isp-kit/metrics" + "isp-config-service/middlewares/sql_metrics" +) + +func SqlOperationMiddleware() httpcli.Middleware { + sqlQueryDuration := metrics.GetOrRegister(metrics.DefaultRegistry, prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Subsystem: "sql", + Name: "query_duration_ms", + Help: "The latencies of sql query", + Objectives: metrics.DefaultObjectives, + }, []string{"operation"})) + return func(next httpcli.RoundTripper) httpcli.RoundTripper { + return httpcli.RoundTripperFunc(func(ctx context.Context, request *httpcli.Request) (*httpcli.Response, error) { + label := sql_metrics.OperationLabelFromContext(ctx) + now := time.Now() + resp, err := next.RoundTrip(ctx, request) + if label != "" { + duration := time.Since(now) + sqlQueryDuration.WithLabelValues(label).Observe(metrics.Milliseconds(duration)) + } + return resp, err + }) + } +} diff --git a/middlewares/sql_metrics/context.go b/middlewares/sql_metrics/context.go new file mode 100644 index 0000000..7278f0b --- /dev/null +++ b/middlewares/sql_metrics/context.go @@ -0,0 +1,20 @@ +package sql_metrics + +import ( + "context" +) + +type tracerContextKey int + +const ( + labelContextKey = tracerContextKey(2) +) + +func OperationLabelToContext(ctx context.Context, label string) context.Context { + return context.WithValue(ctx, labelContextKey, label) +} + +func OperationLabelFromContext(ctx context.Context) string { + value, _ := ctx.Value(labelContextKey).(string) + return value +} diff --git a/middlewares/ws_log.go b/middlewares/ws_log.go new file mode 100644 index 0000000..fe9b2dd --- /dev/null +++ b/middlewares/ws_log.go @@ -0,0 +1,32 @@ +package middlewares + +import ( + "context" + + "github.com/txix-open/etp/v3" + "github.com/txix-open/etp/v3/msg" + "github.com/txix-open/isp-kit/log" + "isp-config-service/helpers" +) + +type EtpMiddleware func(next etp.Handler) etp.Handler + +func EtpLogger(logger log.Logger) EtpMiddleware { + return func(next etp.Handler) etp.Handler { + return etp.HandlerFunc(func(ctx context.Context, conn *etp.Conn, event msg.Event) []byte { + fields := append( + helpers.LogFields(conn), + log.String("event", event.Name), + ) + logger.Debug(ctx, "event received", fields...) + return next.Handle(ctx, conn, event) + }) + } +} + +func EtpChain(root etp.Handler, middlewares ...EtpMiddleware) etp.Handler { + for i := len(middlewares) - 1; i >= 0; i-- { + root = middlewares[i](root) + } + return root +} diff --git a/migrations/20240503120009_init.sql b/migrations/20240503120009_init.sql index 10cbcf2..7463305 100644 --- a/migrations/20240503120009_init.sql +++ b/migrations/20240503120009_init.sql @@ -17,6 +17,7 @@ create table isp_config_service__config data blob not null, version int not null default 1, active int not null default 0, + admin_id int not null default 0, created_at integer not null default (unixepoch()), updated_at integer not null default (unixepoch()), foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade @@ -30,7 +31,7 @@ create table isp_config_service__config_history config_id text not null, data blob not null, version int not null default 1, - admin_id int not null, + admin_id int not null default 0, created_at integer not null default (unixepoch()), foreign key (config_id) references isp_config_service__config (id) on delete cascade on update cascade ); @@ -65,7 +66,7 @@ create table isp_config_service__backend create table isp_config_service__event ( - id integer, + id integer, payload blob not null, created_at integer not null default (unixepoch()), primary key (id desc) diff --git a/repository/backend.go b/repository/backend.go index 09ab5f9..9c90cee 100644 --- a/repository/backend.go +++ b/repository/backend.go @@ -7,6 +7,7 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" "isp-config-service/entity" + "isp-config-service/middlewares/sql_metrics" "isp-config-service/service/rqlite/db" ) @@ -21,6 +22,8 @@ func NewBackend(db db.DB) Backend { } func (r Backend) Upsert(ctx context.Context, backend entity.Backend) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.Upsert") + query, args, err := squirrel.Insert(Table("backend")). Columns("module_id", "address", "version", "lib_version", "module_name", @@ -47,6 +50,8 @@ func (r Backend) Upsert(ctx context.Context, backend entity.Backend) error { } func (r Backend) Delete(ctx context.Context, moduleId string, address string) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.Delete") + query, args, err := squirrel.Delete(Table("backend")). Where(squirrel.Eq{ "module_id": moduleId, @@ -65,6 +70,8 @@ func (r Backend) Delete(ctx context.Context, moduleId string, address string) er } func (r Backend) All(ctx context.Context) ([]entity.Backend, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.All") + result := make([]entity.Backend, 0) query := fmt.Sprintf("SELECT * FROM %s order by created_at desc", Table("backend")) err := r.db.Select(ctx, &result, query) @@ -75,6 +82,8 @@ func (r Backend) All(ctx context.Context) ([]entity.Backend, error) { } func (r Backend) GetByModuleId(ctx context.Context, moduleId string) ([]entity.Backend, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.GetByModuleId") + query, args, err := squirrel.Select("*"). From(Table("backend")). Where(squirrel.Eq{ diff --git a/repository/config.go b/repository/config.go index e26cb6f..97e9d5d 100644 --- a/repository/config.go +++ b/repository/config.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "isp-config-service/entity" "isp-config-service/entity/xtypes" + "isp-config-service/middlewares/sql_metrics" "isp-config-service/service/rqlite/db" ) @@ -21,9 +22,11 @@ func NewConfig(db db.DB) Config { } func (r Config) Insert(ctx context.Context, cfg entity.Config) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Config.Insert") + query, args, err := squirrel.Insert(Table("config")). - Columns("id", "name", "module_id", "data", "version", "active"). - Values(cfg.Id, cfg.Name, cfg.ModuleId, cfg.Data, cfg.Version, cfg.Active). + Columns("id", "name", "module_id", "data", "version", "active", "admin_id"). + Values(cfg.Id, cfg.Name, cfg.ModuleId, cfg.Data, cfg.Version, cfg.Active, cfg.AdminId). Suffix("on conflict (id) do nothing"). ToSql() if err != nil { @@ -39,6 +42,8 @@ func (r Config) Insert(ctx context.Context, cfg entity.Config) error { } func (r Config) GetActive(ctx context.Context, moduleId string) (*entity.Config, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Config.GetActive") + query, args, err := squirrel.Select("*"). From(Table("config")). Where(squirrel.Eq{ @@ -55,6 +60,8 @@ func (r Config) GetActive(ctx context.Context, moduleId string) (*entity.Config, } func (r Config) GetByModuleId(ctx context.Context, moduleId string) ([]entity.Config, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Config.GetByModuleId") + query, args, err := squirrel.Select("*"). From(Table("config")). Where(squirrel.Eq{ @@ -75,6 +82,8 @@ func (r Config) GetByModuleId(ctx context.Context, moduleId string) ([]entity.Co } func (r Config) GetById(ctx context.Context, id string) (*entity.Config, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Config.GetById") + query, args, err := squirrel.Select("*"). From(Table("config")). Where(squirrel.Eq{ @@ -88,10 +97,12 @@ func (r Config) GetById(ctx context.Context, id string) (*entity.Config, error) } func (r Config) DeleteNonActiveById(ctx context.Context, id string) (bool, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Config.DeleteNonActiveById") + query, args, err := squirrel.Delete(Table("config")). Where(squirrel.Eq{ "id": id, - "active": "0", + "active": xtypes.Bool(false), }).ToSql() if err != nil { return false, errors.WithMessage(err, "build query") @@ -110,12 +121,15 @@ func (r Config) DeleteNonActiveById(ctx context.Context, id string) (bool, error } func (r Config) UpdateByVersion(ctx context.Context, cfg entity.Config) (bool, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Config.UpdateByVersion") + query, args, err := squirrel.Update(Table("config")). - SetMap(map[string]interface{}{ + SetMap(map[string]any{ "name": cfg.Name, "data": cfg.Data, "version": squirrel.Expr("version + 1"), "updated_at": squirrel.Expr("unixepoch()"), + "admin_id": cfg.AdminId, }).Where(squirrel.Eq{ "id": cfg.Id, "version": cfg.Version, @@ -137,6 +151,8 @@ func (r Config) UpdateByVersion(ctx context.Context, cfg entity.Config) (bool, e } func (r Config) SetActive(ctx context.Context, configId string, active xtypes.Bool) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Config.SetActive") + query, args, err := squirrel.Update(Table("config")). Set("active", active). Set("updated_at", squirrel.Expr("unixepoch()")). diff --git a/repository/config_history.go b/repository/config_history.go index 47f6cfb..0974d20 100644 --- a/repository/config_history.go +++ b/repository/config_history.go @@ -7,6 +7,7 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" "isp-config-service/entity" + "isp-config-service/middlewares/sql_metrics" "isp-config-service/service/rqlite/db" ) @@ -21,6 +22,8 @@ func NewConfigHistory(db db.DB) ConfigHistory { } func (r ConfigHistory) GetByConfigId(ctx context.Context, configId string) ([]entity.ConfigHistory, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "ConfigHistory.GetByConfigId") + query := fmt.Sprintf(`select * from %s where config_id = ? order by version desc`, Table("config_history")) result := make([]entity.ConfigHistory, 0) err := r.db.Select(ctx, &result, query, configId) @@ -31,6 +34,8 @@ func (r ConfigHistory) GetByConfigId(ctx context.Context, configId string) ([]en } func (r ConfigHistory) Delete(ctx context.Context, id string) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "ConfigHistory.Delete") + query := fmt.Sprintf(`delete from %s where id = ?`, Table("config_history")) _, err := r.db.Exec(ctx, query, id) if err != nil { @@ -40,6 +45,8 @@ func (r ConfigHistory) Delete(ctx context.Context, id string) error { } func (r ConfigHistory) Insert(ctx context.Context, history entity.ConfigHistory) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "ConfigHistory.Insert") + query, args, err := squirrel.Insert(Table("config_history")). Columns("id", "config_id", "data", "version", "admin_id"). Values(history.Id, history.ConfigId, history.Data, history.Version, history.AdminId). diff --git a/repository/config_schema.go b/repository/config_schema.go index 726854e..63f1aa1 100644 --- a/repository/config_schema.go +++ b/repository/config_schema.go @@ -8,6 +8,7 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" "isp-config-service/entity" + "isp-config-service/middlewares/sql_metrics" "isp-config-service/service/rqlite/db" ) @@ -22,6 +23,8 @@ func NewConfigSchema(db db.DB) ConfigSchema { } func (r ConfigSchema) Upsert(ctx context.Context, schema entity.ConfigSchema) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "ConfigSchema.Upsert") + query, args, err := squirrel.Insert(Table("config_schema")). Columns("id", "module_id", "data", "module_version"). Values(schema.Id, schema.ModuleId, schema.Data, schema.ModuleVersion). @@ -41,6 +44,8 @@ func (r ConfigSchema) Upsert(ctx context.Context, schema entity.ConfigSchema) er } func (r ConfigSchema) All(ctx context.Context) ([]entity.ConfigSchema, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "ConfigSchema.All") + result := make([]entity.ConfigSchema, 0) query := fmt.Sprintf("select * from %s order by created_at", Table("config_schema")) err := r.db.Select(ctx, &result, query) @@ -51,6 +56,8 @@ func (r ConfigSchema) All(ctx context.Context) ([]entity.ConfigSchema, error) { } func (r ConfigSchema) GetByModuleId(ctx context.Context, moduleId string) (*entity.ConfigSchema, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "ConfigSchema.GetByModuleId") + query, args, err := squirrel.Select("*"). From(Table("config_schema")). Where(squirrel.Eq{"module_id": moduleId}). diff --git a/repository/event.go b/repository/event.go index 736a484..ad37df0 100644 --- a/repository/event.go +++ b/repository/event.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "isp-config-service/entity" "isp-config-service/entity/xtypes" + "isp-config-service/middlewares/sql_metrics" "isp-config-service/service/rqlite/db" ) @@ -21,6 +22,8 @@ func NewEvent(db db.DB) Event { } func (r Event) Insert(ctx context.Context, event entity.Event) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Event.Insert") + query := fmt.Sprintf("insert into %s (payload) values (?)", Table("event")) _, err := r.db.Exec(ctx, query, event.Payload) if err != nil { @@ -30,6 +33,8 @@ func (r Event) Insert(ctx context.Context, event entity.Event) error { } func (r Event) Get(ctx context.Context, lastEventId int, limit int) ([]entity.Event, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Event.Get") + query := fmt.Sprintf("select * from %s where id > ? order by id desc limit ?", Table("event")) result := make([]entity.Event, 0) err := r.db.Select(ctx, &result, query, lastEventId, limit) @@ -40,6 +45,8 @@ func (r Event) Get(ctx context.Context, lastEventId int, limit int) ([]entity.Ev } func (r Event) DeleteByCreatedAt(ctx context.Context, before xtypes.Time) (int, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Event.DeleteByCreatedAt") + query := fmt.Sprintf("delete from %s where created_at < ?", Table("event")) result, err := r.db.Exec(ctx, query, before) if err != nil { diff --git a/repository/module.go b/repository/module.go index 11dc25c..2fb6323 100644 --- a/repository/module.go +++ b/repository/module.go @@ -7,6 +7,7 @@ import ( "github.com/Masterminds/squirrel" "github.com/pkg/errors" "isp-config-service/entity" + "isp-config-service/middlewares/sql_metrics" "isp-config-service/service/rqlite/db" ) @@ -21,6 +22,8 @@ func NewModule(db db.DB) Module { } func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Module.Upsert") + query, args, err := squirrel.Insert(Table("module")). Columns("id", "name", "last_connected_at"). Values(module.Id, module.Name, squirrel.Expr("unixepoch()")). @@ -43,6 +46,8 @@ func (r Module) SetDisconnectedAtNow( ctx context.Context, moduleId string, ) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Module.SetDisconnectedAtNow") + query, args, err := squirrel.Update(Table("module")). Set("last_disconnected_at", squirrel.Expr("unixepoch()")). Where(squirrel.Eq{"id": moduleId}). @@ -60,6 +65,8 @@ func (r Module) SetDisconnectedAtNow( } func (r Module) GetByNames(ctx context.Context, names []string) ([]entity.Module, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Module.GetByNames") + query, args, err := squirrel.Select("*"). From(Table("module")). Where(squirrel.Eq{ @@ -80,11 +87,15 @@ func (r Module) GetByNames(ctx context.Context, names []string) ([]entity.Module } func (r Module) GetById(ctx context.Context, id string) (*entity.Module, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Module.GetById") + query := fmt.Sprintf("select * from %s where id = ?", Table("module")) return selectRow[entity.Module](ctx, r.db, query, id) } func (r Module) All(ctx context.Context) ([]entity.Module, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Module.All") + result := make([]entity.Module, 0) query := fmt.Sprintf("select * from %s order by name", Table("module")) err := r.db.Select(ctx, &result, query) @@ -95,6 +106,8 @@ func (r Module) All(ctx context.Context) ([]entity.Module, error) { } func (r Module) Delete(ctx context.Context, id string) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Module.Delete") + query, args, err := squirrel.Delete(Table("module")). Where(squirrel.Eq{"id": id}). ToSql() diff --git a/routes/routes.go b/routes/routes.go index 997725d..106ca5a 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -7,8 +7,10 @@ import ( "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/grpc/endpoint" + "github.com/txix-open/isp-kit/log" "isp-config-service/controller" "isp-config-service/controller/api" + mws "isp-config-service/middlewares" ) type Controllers struct { @@ -31,13 +33,22 @@ func GrpcHandler(wrapper endpoint.Wrapper, c Controllers) *grpc.Mux { return muxer } -func BindEtp(etpSrv *etp.Server, c Controllers) { +func BindEtp(etpSrv *etp.Server, c Controllers, logger log.Logger) { + middlewares := []mws.EtpMiddleware{ + mws.EtpLogger(logger), + } etpSrv.OnConnect(c.Module.OnConnect) etpSrv.OnDisconnect(c.Module.OnDisconnect) etpSrv.OnError(c.Module.OnError) - etpSrv.On(cluster.ModuleSendConfigSchema, etp.HandlerFunc(c.Module.OnModuleConfigSchema)) - etpSrv.On(cluster.ModuleSendRequirements, etp.HandlerFunc(c.Module.OnModuleRequirements)) - etpSrv.On(cluster.ModuleReady, etp.HandlerFunc(c.Module.OnModuleReady)) + + onConfigSchema := mws.EtpChain(etp.HandlerFunc(c.Module.OnModuleConfigSchema), middlewares...) + etpSrv.On(cluster.ModuleSendConfigSchema, onConfigSchema) + + onRequirements := mws.EtpChain(etp.HandlerFunc(c.Module.OnModuleRequirements), middlewares...) + etpSrv.On(cluster.ModuleSendRequirements, onRequirements) + + onModuleReady := mws.EtpChain(etp.HandlerFunc(c.Module.OnModuleReady), middlewares...) + etpSrv.On(cluster.ModuleReady, onModuleReady) } func HttpHandler(etpSrv *etp.Server) http.Handler { diff --git a/service/api/config.go b/service/api/config.go index e14fbda..3b9a52d 100644 --- a/service/api/config.go +++ b/service/api/config.go @@ -118,6 +118,7 @@ func (c Config) CreateUpdateConfig( Data: req.Data, Version: 1, Active: false, + AdminId: adminId, } err := c.configRepo.Insert(ctx, config) if err != nil { @@ -140,6 +141,7 @@ func (c Config) CreateUpdateConfig( Name: req.Name, Data: req.Data, Version: req.Version, + AdminId: adminId, } updated, err := c.configRepo.UpdateByVersion(ctx, config) if err != nil { @@ -154,7 +156,7 @@ func (c Config) CreateUpdateConfig( ConfigId: req.Id, Data: oldConfig.Data, Version: oldConfig.Version, - AdminId: adminId, + AdminId: oldConfig.AdminId, } err = c.configHistoryRepo.Insert(ctx, history) if err != nil { diff --git a/service/module/emitter.go b/service/module/emitter.go new file mode 100644 index 0000000..3a0d858 --- /dev/null +++ b/service/module/emitter.go @@ -0,0 +1,59 @@ +package module + +import ( + "context" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/txix-open/etp/v3" + "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/log" + "isp-config-service/helpers" +) + +const ( + emitEventTimeout = 5 * time.Second +) + +type Emitter struct { + logger log.Logger +} + +func NewEmitter(logger log.Logger) Emitter { + return Emitter{ + logger: logger, + } +} + +func (s Emitter) Emit( + ctx context.Context, + conn *etp.Conn, + event string, + data []byte, +) { + ctx, cancel := context.WithTimeout(ctx, emitEventTimeout) + defer cancel() + + fields := append(helpers.LogFields(conn), log.String("event", event)) + if strings.HasSuffix(event, cluster.ModuleConnectionSuffix) { + fields = append(fields, log.ByteString("data", data)) + } + + s.logger.Debug( + ctx, + "emit event", + fields..., + ) + + err := conn.Emit(ctx, event, data) + + if err != nil { + err := errors.WithMessagef( + err, + "emit event '%s', to %s module, connId: %d", + event, helpers.ModuleName(conn), conn.Id(), + ) + s.logger.Error(ctx, err) + } +} diff --git a/service/module/service.go b/service/module/service.go index 84a6e6d..0562aaf 100644 --- a/service/module/service.go +++ b/service/module/service.go @@ -61,6 +61,7 @@ type Service struct { configRepo ConfigRepo configSchemaRepo ConfigSchemaRepo subscriptionService SubscriptionService + emitter Emitter logger log.Logger } @@ -71,6 +72,7 @@ func NewService( configRepo ConfigRepo, configSchemaRepo ConfigSchemaRepo, subscriptionService SubscriptionService, + emitter Emitter, logger log.Logger, ) Service { return Service{ @@ -80,11 +82,14 @@ func NewService( configRepo: configRepo, configSchemaRepo: configSchemaRepo, subscriptionService: subscriptionService, + emitter: emitter, logger: logger, } } func (s Service) OnConnect(ctx context.Context, conn *etp.Conn, moduleName string) error { + s.logger.Info(ctx, "module connected", helpers.LogFields(conn)...) + module := entity.Module{ Id: uuid.NewString(), Name: moduleName, @@ -107,14 +112,17 @@ func (s Service) OnDisconnect( err error, ) error { if isNormalClose { - s.logger.Info(ctx, fmt.Sprintf("module '%s' disconnected", moduleName)) + s.logger.Info( + ctx, + "module disconnected", + helpers.LogFields(conn)..., + ) } else { - message := errors.WithMessagef( + message := errors.WithMessage( err, - "module '%s' unexpectedly disconnected", - moduleName, + "module unexpectedly disconnected", ) - s.logger.Error(ctx, message) + s.logger.Error(ctx, message, helpers.LogFields(conn)...) } moduleId, _ := store.Get[string](conn.Data(), moduleIdKey) @@ -146,13 +154,12 @@ func (s Service) OnDisconnect( return nil } -func (s Service) OnError(ctx context.Context, conn *etp.Conn, moduleName string, err error) { - err = errors.WithMessagef( +func (s Service) OnError(ctx context.Context, conn *etp.Conn, err error) { + err = errors.WithMessage( err, - "unexpected error in communication, module: '%s'", - moduleName, + "unexpected error in communication", ) - s.logger.Error(ctx, err) + s.logger.Error(ctx, err, helpers.LogFields(conn)...) } func (s Service) OnModuleReady( @@ -248,10 +255,7 @@ func (s Service) OnModuleConfigSchema( return errors.WithMessage(err, "get active config") } - err = conn.Emit(ctx, cluster.ConfigSendConfigWhenConnected, config.Data) - if err != nil { - return errors.WithMessage(err, "send event with config") - } + s.emitter.Emit(ctx, conn, cluster.ConfigSendConfigWhenConnected, config.Data) s.subscriptionService.SubscribeToConfigChanges(conn, moduleId) diff --git a/service/startup/service.go b/service/startup/service.go index bf9c516..98e04a2 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -20,6 +20,7 @@ import ( "github.com/txix-open/isp-kit/worker" "isp-config-service/assembly" "isp-config-service/conf" + "isp-config-service/middlewares" "isp-config-service/service/rqlite" "isp-config-service/service/rqlite/db" "isp-config-service/service/rqlite/goose_store" @@ -87,7 +88,11 @@ func (s *Service) Run(ctx context.Context) error { s.logger.Debug(ctx, "is not a leader") } - db, err := db.Open(ctx, s.rqlite.Dsn(), httpclix.Default(httpcli.WithMiddlewares(httpclix.Log(s.logger)))) + db, err := db.Open( + ctx, + s.rqlite.Dsn(), + httpclix.Default(httpcli.WithMiddlewares(middlewares.SqlOperationMiddleware())), + ) if err != nil { return errors.WithMessage(err, "dial to embedded rqlite") } diff --git a/service/subscription/service.go b/service/subscription/service.go index 08a8b60..87545a9 100644 --- a/service/subscription/service.go +++ b/service/subscription/service.go @@ -2,7 +2,6 @@ package subscription import ( "context" - "time" "github.com/pkg/errors" "github.com/txix-open/etp/v3" @@ -28,11 +27,16 @@ type ConfigRepo interface { GetActive(ctx context.Context, moduleId string) (*entity.Config, error) } +type Emitter interface { + Emit(ctx context.Context, conn *etp.Conn, event string, data []byte) +} + type Service struct { moduleRepo ModuleRepo backendRepo BackendRepo configRepo ConfigRepo rooms *etp.Rooms + emitter Emitter logger log.Logger } @@ -41,6 +45,7 @@ func NewService( backendRepo BackendRepo, configRepo ConfigRepo, rooms *etp.Rooms, + emitter Emitter, logger log.Logger, ) Service { return Service{ @@ -48,6 +53,7 @@ func NewService( backendRepo: backendRepo, configRepo: configRepo, rooms: rooms, + emitter: emitter, logger: logger, } } @@ -70,7 +76,7 @@ func (s Service) NotifyConfigChanged(ctx context.Context, moduleId string) error for _, conn := range conns { go func() { - s.emitEvent(ctx, conn, cluster.ConfigSendConfigChanged, config.Data) + s.emitter.Emit(ctx, conn, cluster.ConfigSendConfigChanged, config.Data) }() } @@ -141,7 +147,7 @@ func (s Service) notifyBackendsChanged( for _, conn := range conns { go func() { - s.emitEvent(ctx, conn, cluster.ModuleConnectedEvent(module.Name), data) + s.emitter.Emit(ctx, conn, cluster.ModuleConnectedEvent(module.Name), data) }() } @@ -196,32 +202,9 @@ func (s Service) notifyRoutingChanged(ctx context.Context, event string, conns [ for _, conn := range conns { go func() { - s.emitEvent(ctx, conn, event, data) + s.emitter.Emit(ctx, conn, event, data) }() } return nil } - -const ( - emitEventTimeout = 5 * time.Second -) - -func (s Service) emitEvent( - ctx context.Context, - conn *etp.Conn, - event string, - data []byte, -) { - ctx, cancel := context.WithTimeout(ctx, emitEventTimeout) - defer cancel() - err := conn.Emit(ctx, event, data) - if err != nil { - err := errors.WithMessagef( - err, - "emit event '%s', to %s module, connId: %d", - event, helpers.ModuleName(conn), conn.Id(), - ) - s.logger.Error(ctx, err) - } -} From 125013ba0ae839689bf685bf848e2c8c30e63fcc Mon Sep 17 00:00:00 2001 From: d1slike Date: Tue, 14 May 2024 19:21:19 +0300 Subject: [PATCH 11/40] Implement acceptance test --- conf/config.yml | 4 +- go.mod | 1 + routes/routes.go | 22 ++-- service/rqlite/flags.go | 5 +- service/subscription/service.go | 1 + tests/acceptance_test.go | 200 ++++++++++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 17 deletions(-) create mode 100644 tests/acceptance_test.go diff --git a/conf/config.yml b/conf/config.yml index f0bf664..1b138a4 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -3,10 +3,10 @@ configServiceAddress: port: 9001 grpcOuterAddress: ip: 127.0.0.1 - port: 9001 + port: 9002 grpcInnerAddress: ip: 0.0.0.0 - port: 9001 + port: 9002 moduleName: isp-config-service infraServerPort: 9006 diff --git a/go.mod b/go.mod index bde37b9..fdbe4a6 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/prometheus/client_golang v1.19.0 github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 github.com/rqlite/rqlite/v8 v8.23.4 + github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.17.1 github.com/txix-open/etp/v3 v3.1.0 github.com/txix-open/isp-kit v1.31.2 diff --git a/routes/routes.go b/routes/routes.go index 106ca5a..cf459fa 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -59,47 +59,47 @@ func HttpHandler(etpSrv *etp.Server) http.Handler { func endpointDescriptors(c Controllers) []cluster.EndpointDescriptor { return []cluster.EndpointDescriptor{{ - Path: "module/get_modules_info", + Path: "config/module/get_modules_info", Inner: true, Handler: c.ModuleApi.GetModulesAggregatedInfo, }, { - Path: "module/delete_module", + Path: "config/module/delete_module", Inner: true, Handler: c.ModuleApi.DeleteModule, }, { - Path: "config/get_active_config_by_module_name", + Path: "config/config/get_active_config_by_module_name", Inner: true, Handler: c.ConfigApi.GetActiveConfigByModuleName, }, { - Path: "config/get_configs_by_module_id", + Path: "config/config/get_configs_by_module_id", Inner: true, Handler: c.ConfigApi.GetConfigsByModuleId, }, { - Path: "config/create_update_config", + Path: "config/config/create_update_config", Inner: true, Handler: c.ConfigApi.CreateUpdateConfig, }, { - Path: "config/get_config_by_id", + Path: "config/config/get_config_by_id", Inner: true, Handler: c.ConfigApi.GetConfigById, }, { - Path: "config/mark_config_as_active", + Path: "config/config/mark_config_as_active", Inner: true, Handler: c.ConfigApi.MarkConfigAsActive, }, { - Path: "config/delete_config", + Path: "config/config/delete_config", Inner: true, Handler: c.ConfigApi.DeleteConfigs, }, { - Path: "config/get_all_version", + Path: "config/config/get_all_version", Inner: true, Handler: c.ConfigHistoryApi.GetAllVersion, }, { - Path: "config/delete_version", + Path: "config/config/delete_version", Inner: true, Handler: c.ConfigHistoryApi.DeleteConfigVersion, }, { - Path: "schema/get_by_module_id", + Path: "config/schema/get_by_module_id", Inner: true, Handler: c.ConfigSchemaApi.SchemaByModuleId, }} diff --git a/service/rqlite/flags.go b/service/rqlite/flags.go index e38ca91..9f38ec4 100644 --- a/service/rqlite/flags.go +++ b/service/rqlite/flags.go @@ -427,13 +427,10 @@ type BuildInfo struct { // ParseFlags parses the command line, and returns the configuration. func ParseFlags(name, desc string, build *BuildInfo) (*Config, error) { - if flag.Parsed() { - return nil, fmt.Errorf("command-line flags already parsed") - } config := &Config{} showVersion := false - fs := flag.NewFlagSet(name, flag.ExitOnError) + fs := flag.NewFlagSet(name, flag.ContinueOnError) fs.StringVar(&config.NodeID, "node-id", "", "Unique ID for node. If not set, set to advertised Raft address") fs.StringVar(&config.HTTPAddr, HTTPAddrFlag, "localhost:4001", "HTTP server bind address. To enable HTTPS, set X.509 certificate and key") diff --git a/service/subscription/service.go b/service/subscription/service.go index 87545a9..fa8a73f 100644 --- a/service/subscription/service.go +++ b/service/subscription/service.go @@ -174,6 +174,7 @@ func (s Service) NotifyRoutingChanged(ctx context.Context) error { } func (s Service) notifyRoutingChanged(ctx context.Context, event string, conns []*etp.Conn) error { + ctx = db.NoneConsistency().ToContext(ctx) backends, err := s.backendRepo.All(ctx) if err != nil { return errors.WithMessage(err, "get all backends") diff --git a/tests/acceptance_test.go b/tests/acceptance_test.go new file mode 100644 index 0000000..6af2fe1 --- /dev/null +++ b/tests/acceptance_test.go @@ -0,0 +1,200 @@ +package tests_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/txix-open/isp-kit/bootstrap" + "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/grpc/client" + "github.com/txix-open/isp-kit/json" + "github.com/txix-open/isp-kit/log" + "github.com/txix-open/isp-kit/rc/schema" + "isp-config-service/conf" + "isp-config-service/domain" + "isp-config-service/service/startup" +) + +type clusterEventHandler struct { + receivedConfigs [][]byte + receivedHosts [][]string + receivedRoutes []cluster.RoutingConfig + lock sync.Locker +} + +func newClusterEventHandler() *clusterEventHandler { + return &clusterEventHandler{ + lock: &sync.Mutex{}, + } +} + +func (c *clusterEventHandler) ReceiveConfig(ctx context.Context, remoteConfig []byte) error { + c.lock.Lock() + defer c.lock.Unlock() + c.receivedConfigs = append(c.receivedConfigs, remoteConfig) + return nil +} + +func (c *clusterEventHandler) Upgrade(hosts []string) { + c.lock.Lock() + defer c.lock.Unlock() + c.receivedHosts = append(c.receivedHosts, hosts) +} + +func (c *clusterEventHandler) ReceiveRoutes(ctx context.Context, routes cluster.RoutingConfig) error { + c.lock.Lock() + defer c.lock.Unlock() + c.receivedRoutes = append(c.receivedRoutes, routes) + return nil +} + +func TestAcceptance(t *testing.T) { + t.Parallel() + require := require.New(t) + + err := os.Setenv("APP_CONFIG_PATH", "../conf/config.yml") + require.NoError(err) + err = os.Setenv("DefaultRemoteConfigPath", "../conf/default_remote_config.json") + require.NoError(err) + boot := bootstrap.New("1.0.0", conf.Remote{}, nil) + boot.App.Logger().SetLevel(log.DebugLevel) + boot.MigrationsDir = "../migrations" + dataPath := dataDir(t) + boot.App.Config().Set("rqlite.DataPath", dataPath) + logger := boot.App.Logger() + + startup := startup.New(boot) + t.Cleanup(func() { + for _, closer := range startup.Closers() { + err := closer.Close() + require.NoError(err) + } + }) + err = startup.Run(context.Background()) + require.NoError(err) + time.Sleep(1 * time.Second) + + clientA1 := newClusterClient("A", "10.2.9.1", logger) + go func() { + handler := cluster.NewEventHandler() + err := clientA1.Run(context.Background(), handler) + require.NoError(err) + }() + + clientA2 := newClusterClient("A", "10.2.9.2", logger) + clientA2Ctx, cancelClient2 := context.WithCancel(context.Background()) + go func() { + handler := cluster.NewEventHandler() + err := clientA2.Run(clientA2Ctx, handler) + require.NoError(err) + }() + time.Sleep(2 * time.Second) + + clientB := newClusterClient("B", "10.2.9.2", logger) + eventHandler := newClusterEventHandler() + go func() { + handler := cluster.NewEventHandler(). + RemoteConfigReceiver(eventHandler). + RequireModule("A", eventHandler). + RoutesReceiver(eventHandler) + err := clientB.Run(context.Background(), handler) + require.NoError(err) + }() + time.Sleep(2 * time.Second) + + apiCli, err := client.Default() + require.NoError(err) + apiCli.Upgrade([]string{"127.0.0.1:9002"}) + + activeConfig := domain.Config{} + err = apiCli.Invoke("config/config/get_active_config_by_module_name"). + JsonRequestBody(domain.GetByModuleNameRequest{ModuleName: "B"}). + JsonResponseBody(&activeConfig). + Do(context.Background()) + require.NoError(err) + updateConfigReq := domain.CreateUpdateConfigRequest{ + Id: activeConfig.Id, + Name: activeConfig.Name, + ModuleId: activeConfig.ModuleId, + Version: activeConfig.Version, + Data: []byte(`{"someString": "value"}`), + } + err = apiCli.Invoke("config/config/create_update_config"). + JsonRequestBody(updateConfigReq). + Do(context.Background()) + require.NoError(err) + + cancelClient2() + err = clientA2.Close() + require.NoError(err) + time.Sleep(2 * time.Second) + + require.Len(eventHandler.receivedConfigs, 2) + require.EqualValues([]byte("{}"), eventHandler.receivedConfigs[0]) + + require.Len(eventHandler.receivedHosts, 2) + require.EqualValues([]string{"10.2.9.1:9999", "10.2.9.2:9999"}, eventHandler.receivedHosts[0]) + require.EqualValues([]string{"10.2.9.1:9999"}, eventHandler.receivedHosts[1]) + + require.Len(eventHandler.receivedRoutes, 3) + + statusResponse := make([]domain.ModuleInfo, 0) + err = apiCli.Invoke("config/module/get_modules_info"). + JsonResponseBody(&statusResponse). + Do(context.Background()) + require.NoError(err) + require.Len(statusResponse, 3) + require.EqualValues("A", statusResponse[0].Name) + require.Len(statusResponse[0].Status, 1) + require.EqualValues("B", statusResponse[1].Name) + require.Len(statusResponse[1].Status, 1) + require.NotEmpty(statusResponse[1].ConfigSchema) +} + +func dataDir(t *testing.T) string { + t.Helper() + + data := make([]byte, 6) + _, _ = rand.Read(data) + dir := hex.EncodeToString(data) + t.Cleanup(func() { + _ = os.RemoveAll(dir) + }) + return dir +} + +type testModuleRemoteConfig struct { + SomeString string `validate:"required"` +} + +func newClusterClient( + moduleName string, + host string, + logger log.Logger, +) *cluster.Client { + schema := schema.NewGenerator().Generate(testModuleRemoteConfig{}) + schemaData, err := json.Marshal(schema) + if err != nil { + panic(err) + } + return cluster.NewClient(cluster.ModuleInfo{ + ModuleName: moduleName, + ModuleVersion: "1.0.0", + LibVersion: "1.0.0", + GrpcOuterAddress: cluster.AddressConfiguration{ + IP: host, + Port: "9999", + }, + Endpoints: nil, + }, cluster.ConfigData{ + Version: "1.0.0", + Schema: schemaData, + Config: []byte(`{}`), + }, []string{"127.0.0.1:9001"}, logger) +} From 8275c01d212aec77b703ad2a110ad60193635ecc Mon Sep 17 00:00:00 2001 From: d1slike Date: Wed, 15 May 2024 12:44:39 +0300 Subject: [PATCH 12/40] Align to CI --- .golangci.yml | 2 + assembly/locator.go | 1 + controller/api/config.go | 1 + controller/module.go | 2 +- entity/xtypes/json.go | 11 +++-- entity/xtypes/time.go | 4 +- middlewares/sql.go | 3 +- middlewares/ws_log.go | 1 + migrations/20240503120009_init.sql | 2 +- repository/config_schema.go | 1 + repository/event.go | 2 +- service/api/config.go | 34 ++++++++------- service/api/module.go | 6 ++- service/module/service.go | 6 +-- service/rqlite/db/consistency.go | 2 +- service/rqlite/goose_store/store.go | 23 ++++------- service/rqlite/runner.go | 6 ++- service/startup/service.go | 9 ++-- tests/acceptance_test.go | 62 ++++++---------------------- tests/handler_test.go | 64 +++++++++++++++++++++++++++++ 20 files changed, 144 insertions(+), 98 deletions(-) create mode 100644 tests/handler_test.go diff --git a/.golangci.yml b/.golangci.yml index cdb8788..7f2ac31 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,6 +32,8 @@ linters: - tagalign #не всегда имеет смысл - musttag #у нас собственные правило: не используем теги, если работаем с пакетом isp-kit/json и с внутренними структурами(между нашими сервисами), при этом теги обязательный для всех внених для нас структур - perfsprint #мнимая производительность в угоду читаемости + - gomoddirectives #требует rqlite + - tagliatelle linters-settings: funlen: lines: 80 diff --git a/assembly/locator.go b/assembly/locator.go index c606933..925430c 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -47,6 +47,7 @@ type Config struct { CleanEventWorker *worker.Worker } +//nolint:funlen func (l Locator) Config() Config { moduleRepo := repository.NewModule(l.db) backendRepo := repository.NewBackend(l.db) diff --git a/controller/api/config.go b/controller/api/config.go index 8c40e56..b66d665 100644 --- a/controller/api/config.go +++ b/controller/api/config.go @@ -1,3 +1,4 @@ +//nolint:lll package api import ( diff --git a/controller/module.go b/controller/module.go index 2123515..815f6cf 100644 --- a/controller/module.go +++ b/controller/module.go @@ -13,7 +13,7 @@ import ( ) var ( - ok = []byte("ok") + ok = []byte("ok") //nolint:gochecknoglobals ) type ModuleService interface { diff --git a/entity/xtypes/json.go b/entity/xtypes/json.go index 8d90ccb..29d3dce 100644 --- a/entity/xtypes/json.go +++ b/entity/xtypes/json.go @@ -3,6 +3,7 @@ package xtypes import ( "strconv" + "github.com/pkg/errors" "github.com/txix-open/isp-kit/json" ) @@ -21,7 +22,7 @@ func (l Json[T]) MarshalJSON() ([]byte, error) { func (l *Json[T]) UnmarshalJSON(data []byte) error { s, err := strconv.Unquote(string(data)) if err != nil { - return err + return errors.WithMessage(err, "unquote text") } return l.UnmarshalText([]byte(s)) } @@ -29,11 +30,15 @@ func (l *Json[T]) UnmarshalJSON(data []byte) error { func (l Json[T]) MarshalText() ([]byte, error) { data, err := json.Marshal(l.Value) if err != nil { - return nil, err + return nil, errors.WithMessage(err, "marshal json") } return data, nil } func (l *Json[T]) UnmarshalText(data []byte) error { - return json.Unmarshal(data, &l.Value) + err := json.Unmarshal(data, &l.Value) + if err != nil { + return errors.WithMessage(err, "unmarshal json") + } + return nil } diff --git a/entity/xtypes/time.go b/entity/xtypes/time.go index 6a6dc09..d578271 100644 --- a/entity/xtypes/time.go +++ b/entity/xtypes/time.go @@ -3,6 +3,8 @@ package xtypes import ( "strconv" "time" + + "github.com/pkg/errors" ) type Time time.Time @@ -27,7 +29,7 @@ func (l *Time) UnmarshalText(data []byte) error { } value, err := strconv.Atoi(string(data)) if err != nil { - return err + return errors.WithMessagef(err, "parse int: %s", string(data)) } *l = Time(time.Unix(int64(value), 0)) return nil diff --git a/middlewares/sql.go b/middlewares/sql.go index 5c836ce..3a0d44c 100644 --- a/middlewares/sql.go +++ b/middlewares/sql.go @@ -11,6 +11,7 @@ import ( ) func SqlOperationMiddleware() httpcli.Middleware { + //nolint:promlinter sqlQueryDuration := metrics.GetOrRegister(metrics.DefaultRegistry, prometheus.NewSummaryVec(prometheus.SummaryOpts{ Subsystem: "sql", Name: "query_duration_ms", @@ -26,7 +27,7 @@ func SqlOperationMiddleware() httpcli.Middleware { duration := time.Since(now) sqlQueryDuration.WithLabelValues(label).Observe(metrics.Milliseconds(duration)) } - return resp, err + return resp, err //nolint:wrapcheck }) } } diff --git a/middlewares/ws_log.go b/middlewares/ws_log.go index fe9b2dd..9a7d766 100644 --- a/middlewares/ws_log.go +++ b/middlewares/ws_log.go @@ -24,6 +24,7 @@ func EtpLogger(logger log.Logger) EtpMiddleware { } } +//nolint:ireturn func EtpChain(root etp.Handler, middlewares ...EtpMiddleware) etp.Handler { for i := len(middlewares) - 1; i >= 0; i-- { root = middlewares[i](root) diff --git a/migrations/20240503120009_init.sql b/migrations/20240503120009_init.sql index 7463305..46ddb0c 100644 --- a/migrations/20240503120009_init.sql +++ b/migrations/20240503120009_init.sql @@ -69,7 +69,7 @@ create table isp_config_service__event id integer, payload blob not null, created_at integer not null default (unixepoch()), - primary key (id desc) + primary key (id) ); -- +goose Down diff --git a/repository/config_schema.go b/repository/config_schema.go index 63f1aa1..9bf141e 100644 --- a/repository/config_schema.go +++ b/repository/config_schema.go @@ -69,6 +69,7 @@ func (r ConfigSchema) GetByModuleId(ctx context.Context, moduleId string) (*enti return selectRow[entity.ConfigSchema](ctx, r.db, query, args...) } +//nolint:nilnil func selectRow[T any](ctx context.Context, db db.DB, query string, args ...interface{}) (*T, error) { var result T err := db.SelectRow(ctx, &result, query, args...) diff --git a/repository/event.go b/repository/event.go index ad37df0..b09762a 100644 --- a/repository/event.go +++ b/repository/event.go @@ -35,7 +35,7 @@ func (r Event) Insert(ctx context.Context, event entity.Event) error { func (r Event) Get(ctx context.Context, lastEventId int, limit int) ([]entity.Event, error) { ctx = sql_metrics.OperationLabelToContext(ctx, "Event.Get") - query := fmt.Sprintf("select * from %s where id > ? order by id desc limit ?", Table("event")) + query := fmt.Sprintf("select * from %s where id > ? order by id limit ?", Table("event")) result := make([]entity.Event, 0) err := r.db.Select(ctx, &result, query, lastEventId, limit) if err != nil { diff --git a/service/api/config.go b/service/api/config.go index 3b9a52d..13c646c 100644 --- a/service/api/config.go +++ b/service/api/config.go @@ -111,21 +111,7 @@ func (c Config) CreateUpdateConfig( } if req.Id == "" { - config := entity.Config{ - Id: uuid.NewString(), - Name: req.Name, - ModuleId: req.ModuleId, - Data: req.Data, - Version: 1, - Active: false, - AdminId: adminId, - } - err := c.configRepo.Insert(ctx, config) - if err != nil { - return nil, errors.WithMessage(err, "insert new config") - } - result := configToDto(config, nil) - return &result, nil + return c.insertNewConfig(ctx, adminId, req) } oldConfig, err := c.configRepo.GetById(ctx, req.Id) @@ -239,6 +225,24 @@ func (c Config) DeleteConfig(ctx context.Context, configId string) error { return nil } +func (c Config) insertNewConfig(ctx context.Context, adminId int, req domain.CreateUpdateConfigRequest) (*domain.Config, error) { + config := entity.Config{ + Id: uuid.NewString(), + Name: req.Name, + ModuleId: req.ModuleId, + Data: req.Data, + Version: 1, + Active: false, + AdminId: adminId, + } + err := c.configRepo.Insert(ctx, config) + if err != nil { + return nil, errors.WithMessage(err, "insert new config") + } + result := configToDto(config, nil) + return &result, nil +} + func (c Config) validateConfigUpdate(ctx context.Context, moduleId string, configData []byte) error { schema, err := c.schemaRepo.GetByModuleId(ctx, moduleId) if err != nil { diff --git a/service/api/module.go b/service/api/module.go index f9ae576..3e16139 100644 --- a/service/api/module.go +++ b/service/api/module.go @@ -50,22 +50,24 @@ func (s Module) Status(ctx context.Context) ([]domain.ModuleInfo, error) { modules []entity.Module backends []entity.Backend schemas []entity.ConfigSchema - err error ) group, ctx := errgroup.WithContext(ctx) group.Go(func() error { + var err error modules, err = s.moduleRepo.All(ctx) return errors.WithMessage(err, "get all modules") }) group.Go(func() error { + var err error backends, err = s.backendsRepo.All(ctx) return errors.WithMessage(err, "get all backends") }) group.Go(func() error { + var err error schemas, err = s.schemaRepo.All(ctx) return errors.WithMessage(err, "get all schemas") }) - err = group.Wait() + err := group.Wait() if err != nil { return nil, errors.WithMessage(err, "wait") } diff --git a/service/module/service.go b/service/module/service.go index 0562aaf..94434fc 100644 --- a/service/module/service.go +++ b/service/module/service.go @@ -2,7 +2,7 @@ package module import ( "context" - "crypto/md5" + "crypto/sha256" "encoding/hex" "fmt" @@ -235,8 +235,8 @@ func (s Service) OnModuleConfigSchema( config, err := s.configRepo.GetActive(ctx, moduleId) if config == nil { - md5Sum := md5.Sum([]byte(moduleId)) - initialConfigId := hex.EncodeToString(md5Sum[:]) + hash := sha256.Sum256([]byte(moduleId)) + initialConfigId := hex.EncodeToString(hash[:]) initialConfig := entity.Config{ Id: initialConfigId, Name: helpers.ModuleName(conn), diff --git a/service/rqlite/db/consistency.go b/service/rqlite/db/consistency.go index 673704a..e19ae30 100644 --- a/service/rqlite/db/consistency.go +++ b/service/rqlite/db/consistency.go @@ -8,7 +8,7 @@ import ( type consistencyCtxKey struct{} var ( - consistencyCtxValue = consistencyCtxKey{} + consistencyCtxValue = consistencyCtxKey{} //nolint:gochecknoglobals ) func consistencyFromContext(ctx context.Context) Consistency { diff --git a/service/rqlite/goose_store/store.go b/service/rqlite/goose_store/store.go index 4b9b254..6a3f56a 100644 --- a/service/rqlite/goose_store/store.go +++ b/service/rqlite/goose_store/store.go @@ -56,23 +56,16 @@ func (s Store) GetMigration(ctx context.Context, db database.DBTxConn, version i q := s.querier.GetMigrationByVersion(s.tablename) var result database.GetMigrationResult - rows, err := s.db.QueryContext(ctx, q, version) - if err != nil { - return nil, fmt.Errorf("failed to get migration %d: %w", version, err) + isApplied := float64(0) + err := s.db.QueryRowContext(ctx, q, version).Scan(&result.Timestamp, &isApplied) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %d", database.ErrVersionNotFound, version) } - for rows.Next() { - isApplied := float64(0) - err := rows.Scan(&result.Timestamp, &isApplied) - if err != nil { - return nil, fmt.Errorf("failed to get migration %d: %w", version, err) - } - result.IsApplied = isApplied == 0 - return &result, nil - } - if err := rows.Err(); err != nil { + if err != nil { return nil, fmt.Errorf("failed to get migration %d: %w", version, err) } - return nil, fmt.Errorf("%w: %d", database.ErrVersionNotFound, version) + result.IsApplied = isApplied == 0 + return &result, nil } func (s Store) GetLatestVersion(ctx context.Context, db database.DBTxConn) (int64, error) { @@ -100,7 +93,7 @@ func (s Store) ListMigrations(ctx context.Context, db database.DBTxConn) ([]*dat migrations = append(migrations, &result) } if err := rows.Err(); err != nil { - return nil, err + return nil, errors.WithMessage(err, "fetch rows") } return migrations, nil } diff --git a/service/rqlite/runner.go b/service/rqlite/runner.go index 1c9286b..c77187c 100644 --- a/service/rqlite/runner.go +++ b/service/rqlite/runner.go @@ -62,7 +62,11 @@ func (r *Rqlite) SqlDB() (*sql.DB, error) { return nil, ErrNotRun } - return sql.Open("rqlite", r.Dsn()) + db, err := sql.Open("rqlite", r.Dsn()) + if err != nil { + return nil, errors.WithMessage(err, "open sql db") + } + return db, nil } func (r *Rqlite) Dsn() string { diff --git a/service/startup/service.go b/service/startup/service.go index 98e04a2..93e3ad1 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -38,7 +38,7 @@ type Service struct { clusterCli *cluster.Client logger log.Logger - //initialized in Run + // initialized in Run etpSrv *etp.Server handleEventWorker *worker.Worker cleanEventWorker *worker.Worker @@ -56,6 +56,7 @@ func New(boot *bootstrap.Bootstrap) *Service { } } +// nolint:funlen func (s *Service) Run(ctx context.Context) error { localConfig := conf.Local{} err := s.boot.App.Config().Read(&localConfig) @@ -70,7 +71,7 @@ func (s *Service) Run(ctx context.Context) error { s.boot.Fatal(errors.WithMessage(err, "run embedded rqlite")) } }() - time.Sleep(1 * time.Second) //optimistically wait for store initialization + time.Sleep(1 * time.Second) // optimistically wait for store initialization s.logger.Debug(ctx, fmt.Sprintf("waiting for cluster startup for %s...", waitForLeaderTimeout)) err = s.rqlite.WaitForLeader(waitForLeaderTimeout) @@ -123,7 +124,7 @@ func (s *Service) Run(ctx context.Context) error { s.boot.Fatal(errors.WithMessage(err, "start http server")) } }() - time.Sleep(1 * time.Second) //wait for http start + time.Sleep(1 * time.Second) // wait for http start go func() { err = s.clusterCli.Run(ctx, cluster.NewEventHandler()) @@ -142,7 +143,7 @@ func (s *Service) Closers() []app.Closer { return nil }), app.CloserFunc(func() error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() return s.httpSrv.Shutdown(ctx) }), diff --git a/tests/acceptance_test.go b/tests/acceptance_test.go index 6af2fe1..53e5cf8 100644 --- a/tests/acceptance_test.go +++ b/tests/acceptance_test.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "encoding/hex" "os" - "sync" "testing" "time" @@ -21,47 +20,12 @@ import ( "isp-config-service/service/startup" ) -type clusterEventHandler struct { - receivedConfigs [][]byte - receivedHosts [][]string - receivedRoutes []cluster.RoutingConfig - lock sync.Locker -} - -func newClusterEventHandler() *clusterEventHandler { - return &clusterEventHandler{ - lock: &sync.Mutex{}, - } -} - -func (c *clusterEventHandler) ReceiveConfig(ctx context.Context, remoteConfig []byte) error { - c.lock.Lock() - defer c.lock.Unlock() - c.receivedConfigs = append(c.receivedConfigs, remoteConfig) - return nil -} - -func (c *clusterEventHandler) Upgrade(hosts []string) { - c.lock.Lock() - defer c.lock.Unlock() - c.receivedHosts = append(c.receivedHosts, hosts) -} - -func (c *clusterEventHandler) ReceiveRoutes(ctx context.Context, routes cluster.RoutingConfig) error { - c.lock.Lock() - defer c.lock.Unlock() - c.receivedRoutes = append(c.receivedRoutes, routes) - return nil -} - +//nolint:funlen func TestAcceptance(t *testing.T) { - t.Parallel() require := require.New(t) - err := os.Setenv("APP_CONFIG_PATH", "../conf/config.yml") - require.NoError(err) - err = os.Setenv("DefaultRemoteConfigPath", "../conf/default_remote_config.json") - require.NoError(err) + t.Setenv("APP_CONFIG_PATH", "../conf/config.yml") + t.Setenv("DefaultRemoteConfigPath", "../conf/default_remote_config.json") boot := bootstrap.New("1.0.0", conf.Remote{}, nil) boot.App.Logger().SetLevel(log.DebugLevel) boot.MigrationsDir = "../migrations" @@ -76,7 +40,7 @@ func TestAcceptance(t *testing.T) { require.NoError(err) } }) - err = startup.Run(context.Background()) + err := startup.Run(context.Background()) require.NoError(err) time.Sleep(1 * time.Second) @@ -84,7 +48,7 @@ func TestAcceptance(t *testing.T) { go func() { handler := cluster.NewEventHandler() err := clientA1.Run(context.Background(), handler) - require.NoError(err) + require.NoError(err) //nolint:testifylint }() clientA2 := newClusterClient("A", "10.2.9.2", logger) @@ -92,7 +56,7 @@ func TestAcceptance(t *testing.T) { go func() { handler := cluster.NewEventHandler() err := clientA2.Run(clientA2Ctx, handler) - require.NoError(err) + require.NoError(err) //nolint:testifylint }() time.Sleep(2 * time.Second) @@ -104,7 +68,7 @@ func TestAcceptance(t *testing.T) { RequireModule("A", eventHandler). RoutesReceiver(eventHandler) err := clientB.Run(context.Background(), handler) - require.NoError(err) + require.NoError(err) //nolint:testifylint }() time.Sleep(2 * time.Second) @@ -135,14 +99,14 @@ func TestAcceptance(t *testing.T) { require.NoError(err) time.Sleep(2 * time.Second) - require.Len(eventHandler.receivedConfigs, 2) - require.EqualValues([]byte("{}"), eventHandler.receivedConfigs[0]) + require.Len(eventHandler.ReceivedConfigs(), 2) + require.EqualValues([]byte("{}"), eventHandler.ReceivedConfigs()[0]) - require.Len(eventHandler.receivedHosts, 2) - require.EqualValues([]string{"10.2.9.1:9999", "10.2.9.2:9999"}, eventHandler.receivedHosts[0]) - require.EqualValues([]string{"10.2.9.1:9999"}, eventHandler.receivedHosts[1]) + require.Len(eventHandler.ReceivedHosts(), 2) + require.EqualValues([]string{"10.2.9.1:9999", "10.2.9.2:9999"}, eventHandler.ReceivedHosts()[0]) + require.EqualValues([]string{"10.2.9.1:9999"}, eventHandler.ReceivedHosts()[1]) - require.Len(eventHandler.receivedRoutes, 3) + require.Len(eventHandler.ReceivedRoutes(), 3) statusResponse := make([]domain.ModuleInfo, 0) err = apiCli.Invoke("config/module/get_modules_info"). diff --git a/tests/handler_test.go b/tests/handler_test.go new file mode 100644 index 0000000..795b240 --- /dev/null +++ b/tests/handler_test.go @@ -0,0 +1,64 @@ +package tests_test + +import ( + "context" + "sync" + + "github.com/txix-open/isp-kit/cluster" +) + +type clusterEventHandler struct { + receivedConfigs [][]byte + receivedHosts [][]string + receivedRoutes []cluster.RoutingConfig + lock sync.Locker +} + +func newClusterEventHandler() *clusterEventHandler { + return &clusterEventHandler{ + lock: &sync.Mutex{}, + } +} + +func (c *clusterEventHandler) ReceivedConfigs() [][]byte { + c.lock.Lock() + defer c.lock.Unlock() + return c.receivedConfigs +} + +func (c *clusterEventHandler) ReceivedRoutes() []cluster.RoutingConfig { + c.lock.Lock() + defer c.lock.Unlock() + + return c.receivedRoutes +} + +func (c *clusterEventHandler) ReceivedHosts() [][]string { + c.lock.Lock() + defer c.lock.Unlock() + + return c.receivedHosts +} + +func (c *clusterEventHandler) ReceiveConfig(ctx context.Context, remoteConfig []byte) error { + c.lock.Lock() + defer c.lock.Unlock() + + c.receivedConfigs = append(c.receivedConfigs, remoteConfig) + return nil +} + +func (c *clusterEventHandler) Upgrade(hosts []string) { + c.lock.Lock() + defer c.lock.Unlock() + + c.receivedHosts = append(c.receivedHosts, hosts) +} + +func (c *clusterEventHandler) ReceiveRoutes(ctx context.Context, routes cluster.RoutingConfig) error { + c.lock.Lock() + defer c.lock.Unlock() + + c.receivedRoutes = append(c.receivedRoutes, routes) + return nil +} From 0320131d57a3279b8a57f624a978784ad398594a Mon Sep 17 00:00:00 2001 From: d1slike Date: Wed, 15 May 2024 13:54:40 +0300 Subject: [PATCH 13/40] Add log level --- conf/config.yml | 2 ++ service/startup/service.go | 10 +++++++++- tests/acceptance_test.go | 1 - 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/conf/config.yml b/conf/config.yml index 1b138a4..bd21278 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -11,6 +11,8 @@ moduleName: isp-config-service infraServerPort: 9006 +logLevel: debug + rqlite: DataPath: "data" NodeID: 1 diff --git a/service/startup/service.go b/service/startup/service.go index 93e3ad1..f7a1866 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -58,8 +58,16 @@ func New(boot *bootstrap.Bootstrap) *Service { // nolint:funlen func (s *Service) Run(ctx context.Context) error { + logLevelValue := s.boot.App.Config().Optional().String("logLevel", "info") + var logLevel log.Level + err := logLevel.UnmarshalText([]byte(logLevelValue)) + if err != nil { + return errors.WithMessage(err, "parse log level") + } + s.boot.App.Logger().SetLevel(logLevel) + localConfig := conf.Local{} - err := s.boot.App.Config().Read(&localConfig) + err = s.boot.App.Config().Read(&localConfig) if err != nil { return errors.WithMessage(err, "read local config") } diff --git a/tests/acceptance_test.go b/tests/acceptance_test.go index 53e5cf8..df7dcf4 100644 --- a/tests/acceptance_test.go +++ b/tests/acceptance_test.go @@ -27,7 +27,6 @@ func TestAcceptance(t *testing.T) { t.Setenv("APP_CONFIG_PATH", "../conf/config.yml") t.Setenv("DefaultRemoteConfigPath", "../conf/default_remote_config.json") boot := bootstrap.New("1.0.0", conf.Remote{}, nil) - boot.App.Logger().SetLevel(log.DebugLevel) boot.MigrationsDir = "../migrations" dataPath := dataDir(t) boot.App.Config().Set("rqlite.DataPath", dataPath) From 9bc9655c3de6278993377285889245c5a5b5299d Mon Sep 17 00:00:00 2001 From: d1slike Date: Wed, 15 May 2024 14:39:25 +0300 Subject: [PATCH 14/40] Add Dockerfile --- Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cd532e3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.22-alpine3.19 as builder +WORKDIR /build +ARG version +ENV version_env=$version +ARG app_name +ENV app_name_env=$app_name +COPY . . +RUN apk update && apk upgrade && apk add --no-cache sqlite-dev +RUN go build -ldflags="-X 'main.version=$version_env'" -o /main . + +FROM alpine:3.19 +WORKDIR /app +ARG app_name +ENV app_name_env=$app_name +COPY --from=builder main /app/$app_name_env +COPY /conf/config.yml /app/config.yml +COPY /conf/default_remote_config.json* /app/default_remote_config.json +COPY /migrations /app/migrations +ENTRYPOINT /app/$app_name_env From d7d17e7f1395b22796a8dfda0f8206783455a23f Mon Sep 17 00:00:00 2001 From: d1slike Date: Wed, 15 May 2024 14:42:52 +0300 Subject: [PATCH 15/40] Add Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cd532e3..ac056cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ENV version_env=$version ARG app_name ENV app_name_env=$app_name COPY . . -RUN apk update && apk upgrade && apk add --no-cache sqlite-dev +RUN apk update && apk upgrade && apk add --no-cache gcc musl-dev RUN go build -ldflags="-X 'main.version=$version_env'" -o /main . FROM alpine:3.19 From 2228e02770769b214d7be1cc77e6ed22e3d630db Mon Sep 17 00:00:00 2001 From: d1slike Date: Thu, 16 May 2024 20:27:40 +0300 Subject: [PATCH 16/40] Implement missing endpoint --- controller/api/module.go | 22 ++++++++++++-- docs/swagger.yaml | 48 +++++++++++++++++++---------- domain/response.go | 15 +++++++-- repository/config.go | 2 +- routes/routes.go | 6 +++- service/api/module.go | 62 ++++++++++++++++++++++++++++++++------ service/module/service.go | 6 ++-- service/startup/service.go | 3 +- 8 files changed, 128 insertions(+), 36 deletions(-) diff --git a/controller/api/module.go b/controller/api/module.go index d494e0e..ab8e6e8 100644 --- a/controller/api/module.go +++ b/controller/api/module.go @@ -10,6 +10,7 @@ import ( type ModuleService interface { Status(ctx context.Context) ([]domain.ModuleInfo, error) Delete(ctx context.Context, id string) error + Connections(ctx context.Context) ([]domain.Connection, error) } type Module struct { @@ -22,7 +23,7 @@ func NewModule(service ModuleService) Module { } } -// GetModulesAggregatedInfo +// Status // @Summary Метод полчения полной информации о состоянии модулей // @Description Возвращает полное состояние всех модулей в кластере (схема конфигурации, подключенные экземпляры) // @Tags Модули @@ -31,7 +32,7 @@ func NewModule(service ModuleService) Module { // @Success 200 {array} domain.ModuleInfo // @Failure 500 {object} apierrors.Error // @Router /module/get_modules_info [POST] -func (c Module) GetModulesAggregatedInfo(ctx context.Context) ([]domain.ModuleInfo, error) { +func (c Module) Status(ctx context.Context) ([]domain.ModuleInfo, error) { modulesInfo, err := c.service.Status(ctx) if err != nil { return nil, apierrors.NewInternalServiceError(err) @@ -64,6 +65,23 @@ func (c Module) DeleteModule(ctx context.Context, identities []string) (*domain. }, nil } +// Connections +// @Summary Метод получения маршрутов +// @Description Возвращает все доступные роуты +// @Tags Модули +// @Accept json +// @Produce json +// @Success 200 {array} domain.Connection +// @Failure 500 {object} apierrors.Error +// @Router /routing/get_routes [POST] +func (c Module) Connections(ctx context.Context) ([]domain.Connection, error) { + connections, err := c.service.Connections(ctx) + if err != nil { + return nil, apierrors.NewInternalServiceError(err) + } + return connections, nil +} + func getSingleId(identities []string) (string, error) { if len(identities) == 0 { return "", apierrors.NewBusinessError(domain.ErrorCodeBadRequest, "at least one id is required", nil) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6ccf683..b87c30c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,24 +1,12 @@ basePath: /api/config definitions: - cluster.AddressConfiguration: + domain.Address: properties: ip: type: string port: type: string type: object - cluster.EndpointDescriptor: - properties: - extra: - additionalProperties: {} - type: object - inner: - type: boolean - path: - type: string - userAuthRequired: - type: boolean - type: object domain.Config: properties: active: @@ -78,10 +66,10 @@ definitions: domain.Connection: properties: address: - $ref: '#/definitions/cluster.AddressConfiguration' + $ref: '#/definitions/domain.Address' endpoints: items: - $ref: '#/definitions/cluster.EndpointDescriptor' + $ref: '#/definitions/domain.EndpointDescriptor' type: array establishedAt: type: string @@ -113,6 +101,13 @@ definitions: deleted: type: integer type: object + domain.EndpointDescriptor: + properties: + inner: + type: boolean + path: + type: string + type: object domain.GetByModuleIdRequest: properties: moduleId: @@ -165,7 +160,7 @@ info: license: name: GNU GPL v3.0 title: isp-config-service - version: 1.0.0 + version: 3.0.0 paths: /config/create_update_config: post: @@ -466,6 +461,27 @@ paths: summary: Метод полчения полной информации о состоянии модулей tags: - Модули + /routing/get_routes: + post: + consumes: + - application/json + description: Возвращает все доступные роуты + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/domain.Connection' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/github_com_txix-open_isp-kit_grpc_apierrors.Error' + summary: Метод получения маршрутов + tags: + - Модули /schema/get_by_module_id: post: consumes: diff --git a/domain/response.go b/domain/response.go index bc92cfe..c8ffe37 100644 --- a/domain/response.go +++ b/domain/response.go @@ -3,7 +3,6 @@ package domain import ( "time" - "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/json" ) @@ -25,11 +24,21 @@ type ModuleInfo struct { type Connection struct { LibVersion string Version string - Address cluster.AddressConfiguration - Endpoints []cluster.EndpointDescriptor + Address Address + Endpoints []EndpointDescriptor EstablishedAt time.Time } +type Address struct { + Ip string + Port string +} + +type EndpointDescriptor struct { + Path string + Inner bool +} + type Config struct { Id string Name string diff --git a/repository/config.go b/repository/config.go index 97e9d5d..0192a91 100644 --- a/repository/config.go +++ b/repository/config.go @@ -48,7 +48,7 @@ func (r Config) GetActive(ctx context.Context, moduleId string) (*entity.Config, From(Table("config")). Where(squirrel.Eq{ "module_id": moduleId, - "active": "1", + "active": xtypes.Bool(true), }).OrderBy("version DESC"). Limit(1). ToSql() diff --git a/routes/routes.go b/routes/routes.go index cf459fa..c631c0e 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -61,11 +61,15 @@ func endpointDescriptors(c Controllers) []cluster.EndpointDescriptor { return []cluster.EndpointDescriptor{{ Path: "config/module/get_modules_info", Inner: true, - Handler: c.ModuleApi.GetModulesAggregatedInfo, + Handler: c.ModuleApi.Status, }, { Path: "config/module/delete_module", Inner: true, Handler: c.ModuleApi.DeleteModule, + }, { + Path: "config/routing/get_routes", + Inner: true, + Handler: c.ModuleApi.Connections, }, { Path: "config/config/get_active_config_by_module_name", Inner: true, diff --git a/service/api/module.go b/service/api/module.go index 3e16139..b5966cb 100644 --- a/service/api/module.go +++ b/service/api/module.go @@ -1,7 +1,9 @@ package api import ( + "cmp" "context" + "slices" "time" "github.com/pkg/errors" @@ -74,16 +76,9 @@ func (s Module) Status(ctx context.Context) ([]domain.ModuleInfo, error) { connections := make(map[string][]domain.Connection, len(modules)) for _, backend := range backends { - addr, err := helpers.SplitAddress(backend) + conn, err := s.backendToDto(backend) if err != nil { - return nil, errors.WithMessage(err, "split address") - } - conn := domain.Connection{ - LibVersion: backend.LibVersion, - Version: backend.Version, - Address: addr, - Endpoints: backend.Endpoints.Value, - EstablishedAt: time.Time(backend.UpdatedAt), + return nil, errors.WithMessage(err, "backend to dto") } connections[backend.ModuleId] = append(connections[backend.ModuleId], conn) } @@ -118,3 +113,52 @@ func (s Module) Delete(ctx context.Context, id string) error { } return nil } + +func (s Module) Connections(ctx context.Context) ([]domain.Connection, error) { + backends, err := s.backendsRepo.All(ctx) + if err != nil { + return nil, errors.WithMessage(err, "get all backends") + } + + connections := make([]domain.Connection, 0, len(backends)) + for _, backend := range backends { + conn, err := s.backendToDto(backend) + if err != nil { + return nil, errors.WithMessage(err, "backend to dto") + } + connections = append(connections, conn) + } + + return connections, nil +} + +func (s Module) backendToDto(backend entity.Backend) (domain.Connection, error) { + addr, err := helpers.SplitAddress(backend) + if err != nil { + return domain.Connection{}, errors.WithMessage(err, "split address") + } + + endpointsDescriptors := make([]domain.EndpointDescriptor, 0) + for _, desc := range backend.Endpoints.Value { + endpointsDescriptors = append(endpointsDescriptors, domain.EndpointDescriptor{ + Path: desc.Path, + Inner: desc.Inner, + }) + } + slices.SortFunc(endpointsDescriptors, func(a, b domain.EndpointDescriptor) int { + return cmp.Compare(a.Path, b.Path) + }) + + conn := domain.Connection{ + LibVersion: backend.LibVersion, + Version: backend.Version, + Address: domain.Address{ + Ip: addr.IP, + Port: addr.Port, + }, + Endpoints: endpointsDescriptors, + EstablishedAt: time.Time(backend.UpdatedAt), + } + + return conn, nil +} diff --git a/service/module/service.go b/service/module/service.go index 94434fc..722ea61 100644 --- a/service/module/service.go +++ b/service/module/service.go @@ -234,6 +234,9 @@ func (s Service) OnModuleConfigSchema( } config, err := s.configRepo.GetActive(ctx, moduleId) + if err != nil { + return errors.WithMessage(err, "get active config") + } if config == nil { hash := sha256.Sum256([]byte(moduleId)) initialConfigId := hex.EncodeToString(hash[:]) @@ -251,9 +254,6 @@ func (s Service) OnModuleConfigSchema( } config = &initialConfig } - if err != nil { - return errors.WithMessage(err, "get active config") - } s.emitter.Emit(ctx, conn, cluster.ConfigSendConfigWhenConnected, config.Data) diff --git a/service/startup/service.go b/service/startup/service.go index f7a1866..b2924dd 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -17,6 +17,7 @@ import ( "github.com/txix-open/isp-kit/http/httpcli" "github.com/txix-open/isp-kit/http/httpclix" "github.com/txix-open/isp-kit/log" + "github.com/txix-open/isp-kit/observability/sentry" "github.com/txix-open/isp-kit/worker" "isp-config-service/assembly" "isp-config-service/conf" @@ -52,7 +53,7 @@ func New(boot *bootstrap.Bootstrap) *Service { grpcSrv: grpc.DefaultServer(), httpSrv: http.NewServer(boot.App.Logger()), clusterCli: boot.ClusterCli, - logger: boot.App.Logger(), + logger: sentry.WrapErrorLogger(boot.App.Logger(), boot.SentryHub), } } From bf024474d156d3a8161df3864caa243401fa65d9 Mon Sep 17 00:00:00 2001 From: d1slike Date: Thu, 16 May 2024 20:31:18 +0300 Subject: [PATCH 17/40] Implement missing endpoint --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fdbe4a6..868ba78 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/google/uuid v1.6.0 github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.20.0 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 github.com/rqlite/rqlite/v8 v8.23.4 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 5b96323..1bc8b96 100644 --- a/go.sum +++ b/go.sum @@ -167,6 +167,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From a6fa3eee8ab839ed5bdbc8700807918c4a16ded3 Mon Sep 17 00:00:00 2001 From: d1slike Date: Fri, 17 May 2024 11:37:03 +0300 Subject: [PATCH 18/40] Reimplement disovery service --- service/subscription/rooms.go | 4 +-- service/subscription/service.go | 64 ++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/service/subscription/rooms.go b/service/subscription/rooms.go index 8b1d9f1..3fc3772 100644 --- a/service/subscription/rooms.go +++ b/service/subscription/rooms.go @@ -8,8 +8,8 @@ func ConfigChangingRoom(moduleId string) string { return fmt.Sprintf("config_changing_room.%s", moduleId) } -func BackendsChangingRoom(moduleId string) string { - return fmt.Sprintf("backends_changing_room.%s", moduleId) +func BackendsChangingRoom(moduleName string) string { + return fmt.Sprintf("backends_changing_room.%s", moduleName) } func RoutingChangingRoom() string { diff --git a/service/subscription/service.go b/service/subscription/service.go index fa8a73f..e7a3361 100644 --- a/service/subscription/service.go +++ b/service/subscription/service.go @@ -89,12 +89,23 @@ func (s Service) SubscribeToBackendsChanges(ctx context.Context, conn *etp.Conn, if err != nil { return errors.WithMessage(err, "get modules by names") } + moduleByName := make(map[string]entity.Module) + for _, module := range modules { + moduleByName[module.Name] = module + } roomsToJoin := make([]string, 0) - for _, module := range modules { - roomsToJoin = append(roomsToJoin, BackendsChangingRoom(module.Id)) + conns := []*etp.Conn{conn} + for _, moduleName := range requiredModuleNames { + roomsToJoin = append(roomsToJoin, BackendsChangingRoom(moduleName)) go func() { - err := s.notifyBackendsChanged(ctx, module.Id, []*etp.Conn{conn}) + module, ok := moduleByName[moduleName] + if !ok { + s.emitModuleConnectedEvent(ctx, conns, moduleName, []cluster.AddressConfiguration{}) + return + } + + err := s.notifyBackendsChanged(ctx, module.Id, moduleName, conns) if err != nil { s.logger.Error(ctx, errors.WithMessage(err, "notify backends changed")) } @@ -107,26 +118,30 @@ func (s Service) SubscribeToBackendsChanges(ctx context.Context, conn *etp.Conn, } func (s Service) NotifyBackendsChanged(ctx context.Context, moduleId string) error { - conns := s.rooms.ToBroadcast(BackendsChangingRoom(moduleId)) + ctx = db.NoneConsistency().ToContext(ctx) + module, err := s.moduleRepo.GetById(ctx, moduleId) + if err != nil { + return errors.WithMessage(err, "get module by id") + } + if module == nil { + return errors.Errorf("unknown module: %s", moduleId) + } + + conns := s.rooms.ToBroadcast(BackendsChangingRoom(module.Name)) if len(conns) == 0 { return nil } - return s.notifyBackendsChanged(ctx, moduleId, conns) + + return s.notifyBackendsChanged(ctx, moduleId, module.Name, conns) } func (s Service) notifyBackendsChanged( ctx context.Context, moduleId string, + moduleName string, conns []*etp.Conn, ) error { ctx = db.NoneConsistency().ToContext(ctx) - module, err := s.moduleRepo.GetById(ctx, moduleId) - if err != nil { - return errors.WithMessage(err, "get module by id") - } - if module == nil { - return errors.Errorf("unknown module: %s", moduleId) - } backends, err := s.backendRepo.GetByModuleId(ctx, moduleId) if err != nil { return errors.WithMessage(err, "get backends by module id") @@ -140,16 +155,8 @@ func (s Service) notifyBackendsChanged( } addresses = append(addresses, addr) } - data, err := json.Marshal(addresses) - if err != nil { - return errors.WithMessage(err, "marshal addresses") - } - for _, conn := range conns { - go func() { - s.emitter.Emit(ctx, conn, cluster.ModuleConnectedEvent(module.Name), data) - }() - } + s.emitModuleConnectedEvent(ctx, conns, moduleName, addresses) return nil } @@ -209,3 +216,18 @@ func (s Service) notifyRoutingChanged(ctx context.Context, event string, conns [ return nil } + +func (s Service) emitModuleConnectedEvent( + ctx context.Context, + conns []*etp.Conn, + moduleName string, + addresses []cluster.AddressConfiguration, +) { + data, _ := json.Marshal(addresses) + + for _, conn := range conns { + go func() { + s.emitter.Emit(ctx, conn, cluster.ModuleConnectedEvent(moduleName), data) + }() + } +} From c3d626f0747bc70db85ae21339e67732c9f54489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D1=83=D0=B7=D1=8C=D0=BC=D0=B8=D0=BD=20=D0=98=D0=B3?= =?UTF-8?q?=D0=BE=D1=80=D1=8C=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0=D0=BD?= =?UTF-8?q?=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Mon, 20 May 2024 15:40:03 +0000 Subject: [PATCH 19/40] Update config.yml --- conf/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/config.yml b/conf/config.yml index bd21278..b9331a8 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -17,8 +17,8 @@ rqlite: DataPath: "data" NodeID: 1 BootstrapExpect: 1 - RaftAddr: "127.0.0.1:9003" - JoinAddrs: "127.0.0.1:9003" + RaftAddr: "127.0.0.1:9008" + JoinAddrs: "127.0.0.1:9008" FKConstraints: true AutoVacInterval: 12h From 13c5ed92f427a6d22609f1d64ebfe9c96d82311a Mon Sep 17 00:00:00 2001 From: d1slike Date: Fri, 24 May 2024 13:25:10 +0300 Subject: [PATCH 20/40] Do not check is response is array (fix panic) --- cmd/migrate/go.mod | 18 ++++++++++++++++++ cmd/migrate/main.go | 11 +++++++++++ service/rqlite/db/db.go | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 cmd/migrate/go.mod create mode 100644 cmd/migrate/main.go diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod new file mode 100644 index 0000000..240ddda --- /dev/null +++ b/cmd/migrate/go.mod @@ -0,0 +1,18 @@ +module isp-config-service/migrate + +go 1.22 + +require github.com/txix-open/isp-kit v1.31.2 + +require ( + github.com/iancoleman/strcase v0.3.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/text v0.14.0 // indirect +) diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 0000000..a0bd1f8 --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "context" + + "github.com/txix-open/isp-kit/db" +) + +func main() { + db.Open(context.Background(), "") +} diff --git a/service/rqlite/db/db.go b/service/rqlite/db/db.go index d89cc2e..203c8b8 100644 --- a/service/rqlite/db/db.go +++ b/service/rqlite/db/db.go @@ -84,7 +84,7 @@ func (d Adapter) SelectRow(ctx context.Context, ptr any, query string, args ...a body, _ := httpResp.Body() rows := gjson.GetBytes(body, "results.0.rows") elems := rows.Array() - if rows.IsArray() && len(elems) == 0 { + if len(elems) == 0 { return sql.ErrNoRows } err = json.Unmarshal([]byte(elems[0].Raw), ptr) From fb21d434251de017080791a1587284842c9d3acc Mon Sep 17 00:00:00 2001 From: d1slike Date: Fri, 24 May 2024 16:26:00 +0300 Subject: [PATCH 21/40] Use idempotent module id --- cmd/migrate/entities.go | 43 +++++++++ cmd/migrate/go.mod | 33 ++++++- cmd/migrate/go.sum | 149 ++++++++++++++++++++++++++++++++ cmd/migrate/main.go | 136 ++++++++++++++++++++++++++++- cmd/migrate/migration/runner.go | 86 ++++++++++++++++++ migrations/embed.go | 10 +++ repository/module.go | 12 ++- service/module/service.go | 16 ++-- 8 files changed, 467 insertions(+), 18 deletions(-) create mode 100644 cmd/migrate/entities.go create mode 100644 cmd/migrate/go.sum create mode 100644 cmd/migrate/migration/runner.go create mode 100644 migrations/embed.go diff --git a/cmd/migrate/entities.go b/cmd/migrate/entities.go new file mode 100644 index 0000000..519f41a --- /dev/null +++ b/cmd/migrate/entities.go @@ -0,0 +1,43 @@ +package main + +import ( + "time" + + "github.com/txix-open/isp-kit/json" +) + +type Module struct { + Id string + Name string + LastConnectedAt *time.Time + LastDisconnectedAt *time.Time + CreatedAt time.Time +} + +type Config struct { + Id string + ModuleId string + Name string + Version int + Active bool + Data json.RawMessage + CreatedAt time.Time + UpdatedAt time.Time +} + +type ConfigHistory struct { + Id string + ConfigId string + ConfigVersion int + Data json.RawMessage + CreatedAt time.Time +} + +type ConfigSchema struct { + Id string + ModuleId string + Version string + Schema json.RawMessage + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index 240ddda..7d214ce 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -2,17 +2,44 @@ module isp-config-service/migrate go 1.22 -require github.com/txix-open/isp-kit v1.31.2 +replace isp-config-service => ../.. require ( + github.com/jmoiron/sqlx v1.4.0 + github.com/mattn/go-sqlite3 v1.14.22 + github.com/pkg/errors v0.9.1 + github.com/pressly/goose/v3 v3.20.0 + github.com/txix-open/isp-kit v1.31.2 + isp-config-service v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/Masterminds/squirrel v1.5.4 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/jmoiron/sqlx v1.4.0 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/sethvargo/go-retry v0.2.4 // indirect + github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/txix-open/bellows v1.2.0 // indirect + github.com/txix-open/etp/v3 v3.1.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/text v0.14.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + nhooyr.io/websocket v1.8.11 // indirect ) diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum new file mode 100644 index 0000000..f8d1a45 --- /dev/null +++ b/cmd/migrate/go.sum @@ -0,0 +1,149 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= +github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA= +github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0= +github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= +github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgjeY= +github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o= +github.com/txix-open/etp/v3 v3.1.0 h1:EQdjm0x2K6Dr6W8oRvs/dSU4rnw+g9o1THahIxLBC3E= +github.com/txix-open/etp/v3 v3.1.0/go.mod h1:aXqeKBdu55VOtmzx+PMlDAcz6eTNBMt3vP3NkiE1qj0= +github.com/txix-open/isp-kit v1.31.2 h1:qt6Om1wF+vpbIz01jQix9L20h9ftwn3/SasgO1RT9T0= +github.com/txix-open/isp-kit v1.31.2/go.mod h1:t6/NHdK5JvpIiQU4+NUjky4jLbo7Bz/4hzpoBxZt64U= +github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= +github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= +modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= +modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= +nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index a0bd1f8..f012e70 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -1,11 +1,143 @@ package main import ( - "context" + "fmt" + "os" + "time" + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/app" "github.com/txix-open/isp-kit/db" + "isp-config-service/entity" + "isp-config-service/entity/xtypes" + "isp-config-service/migrate/migration" + "isp-config-service/migrations" + "isp-config-service/repository" ) func main() { - db.Open(context.Background(), "") + if len(os.Args) == 1 { + fmt.Println("usage: migrate postgres://username:password@localhost:5432/database_name?search_path=config_service") + os.Exit(1) + } + app, err := app.New() + if err != nil { + panic(err) + } + logger := app.Logger() + ctx := app.Context() + + pgDb, err := db.Open(ctx, os.Args[1]) + if err != nil { + panic(err) + } + defer pgDb.Close() + + fileName := fmt.Sprintf("db_%d.sqlite", time.Now().Unix()) + dbClient := &db.Client{} + dbClient.DB, err = sqlx.Open("sqlite3", fileName) + dbClient.DB.MapperFunc(db.ToSnakeCase) + if err != nil { + panic(errors.WithMessage(err, "err opening db")) + } + defer dbClient.Close() + + mr := migration.NewRunner(migration.DialectSqlite3, migrations.Migrations, logger) + err = mr.Run(ctx, dbClient.DB.DB) + if err != nil { + panic(errors.WithMessage(err, "err connecting to db")) + } + + moduleRepo := repository.NewModule(dbClient) + configRepo := repository.NewConfig(dbClient) + configSchemaRepo := repository.NewConfigSchema(dbClient) + configHistoryRepo := repository.NewConfigHistory(dbClient) + + modules := make([]Module, 0) + err = pgDb.Select(ctx, &modules, "select * from modules") + if err != nil { + panic(errors.WithMessage(err, "err selecting modules")) + } + for _, module := range modules { + result := entity.Module{ + Id: module.Id, + Name: module.Name, + CreatedAt: xtypes.Time(module.CreatedAt), + } + err := moduleRepo.Upsert(ctx, result) + if err != nil { + panic(errors.WithMessagef(err, "err upserting module: %s", module.Name)) + } + } + logger.Info(ctx, "modules migrated") + + configs := make([]Config, 0) + err = pgDb.Select(ctx, &configs, "select * from configs") + if err != nil { + panic(errors.WithMessage(err, "err selecting configs")) + } + for _, config := range configs { + result := entity.Config{ + Id: config.Id, + Name: config.Name, + ModuleId: config.ModuleId, + Data: config.Data, + Version: config.Version, + Active: xtypes.Bool(config.Active), + AdminId: 0, + CreatedAt: xtypes.Time(config.CreatedAt), + UpdatedAt: xtypes.Time(config.UpdatedAt), + } + err := configRepo.Insert(ctx, result) + if err != nil { + panic(errors.WithMessagef(err, "err inserting config: %s", config.Id)) + } + } + logger.Info(ctx, "configs migrated") + + schemas := make([]ConfigSchema, 0) + err = pgDb.Select(ctx, &schemas, "select * from config_schemas") + if err != nil { + panic(errors.WithMessage(err, "err selecting schemas")) + } + for _, schema := range schemas { + result := entity.ConfigSchema{ + Id: schema.Id, + ModuleId: schema.ModuleId, + Data: schema.Schema, + ModuleVersion: schema.Version, + CreatedAt: xtypes.Time(schema.CreatedAt), + UpdatedAt: xtypes.Time(schema.UpdatedAt), + } + err := configSchemaRepo.Upsert(ctx, result) + if err != nil { + panic(errors.WithMessagef(err, "err upserting config_schema: %s", schema.Id)) + } + } + logger.Info(ctx, "config schemas migrated") + + configHistories := make([]ConfigHistory, 0) + err = pgDb.Select(ctx, &configHistories, "select * from version_config") + if err != nil { + panic(errors.WithMessage(err, "err selecting configs")) + } + for _, history := range configHistories { + result := entity.ConfigHistory{ + Id: history.Id, + ConfigId: history.ConfigId, + Data: history.Data, + Version: history.ConfigVersion, + AdminId: 0, + CreatedAt: xtypes.Time(history.CreatedAt), + } + err := configHistoryRepo.Insert(ctx, result) + if err != nil { + panic(errors.WithMessagef(err, "err inserting config_history: %s", history.Id)) + } + } + logger.Info(ctx, "config history migrated") + + logger.Info(ctx, "done!") } diff --git a/cmd/migrate/migration/runner.go b/cmd/migrate/migration/runner.go new file mode 100644 index 0000000..82bb783 --- /dev/null +++ b/cmd/migrate/migration/runner.go @@ -0,0 +1,86 @@ +package migration + +import ( + "context" + "database/sql" + "fmt" + "io/fs" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/pressly/goose/v3" + "github.com/txix-open/isp-kit/log" +) + +const ( + DialectPostgreSQL = goose.DialectPostgres + DialectSqlite3 = goose.DialectSQLite3 + DialectClickhouse = goose.DialectClickHouse +) + +type Runner struct { + dialect goose.Dialect + fs fs.FS + logger log.Logger +} + +func NewRunner( + dialect goose.Dialect, + fs fs.FS, + logger log.Logger, +) Runner { + return Runner{ + dialect: dialect, + fs: fs, + logger: logger, + } +} + +func (r Runner) Run(ctx context.Context, db *sql.DB, gooseOpts ...goose.ProviderOption) error { + ctx = log.ToContext(ctx, log.String("worker", "goose_db_migration")) + + provider, err := goose.NewProvider(r.dialect, db, r.fs, gooseOpts...) + if err != nil { + return errors.WithMessage(err, "get goose provider") + } + + dbVersion, err := provider.GetDBVersion(ctx) + if err != nil { + return errors.WithMessage(err, "get db version") + } + r.logger.Info(ctx, fmt.Sprintf("current db version: %d", dbVersion)) + + migrations, err := provider.Status(ctx) + if err != nil { + return errors.WithMessage(err, "get status migrations") + } + r.logger.Info(ctx, "print migration list") + if len(migrations) == 0 { + r.logger.Info(ctx, "no migrations") + } + for _, migration := range migrations { + appliedAt := "Pending" + if !migration.AppliedAt.IsZero() { + appliedAt = migration.AppliedAt.Format(time.RFC3339) + } + msg := fmt.Sprintf( + "migration: %s %s %s", + filepath.Base(migration.Source.Path), + strings.ToUpper(string(migration.State)), + appliedAt, + ) + r.logger.Info(ctx, msg) + } + + result, err := provider.Up(ctx) + if err != nil { + return errors.WithMessage(err, "apply pending migrations") + } + for _, migrationResult := range result { + r.logger.Info(ctx, fmt.Sprintf("applied migration: %s", migrationResult.String())) + } + + return nil +} diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 0000000..753aaf1 --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,10 @@ +package migrations + +import ( + "embed" +) + +var ( + //go:embed * + Migrations embed.FS +) diff --git a/repository/module.go b/repository/module.go index 2fb6323..0bc84f3 100644 --- a/repository/module.go +++ b/repository/module.go @@ -21,25 +21,23 @@ func NewModule(db db.DB) Module { } } -func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error) { +func (r Module) Upsert(ctx context.Context, module entity.Module) error { ctx = sql_metrics.OperationLabelToContext(ctx, "Module.Upsert") query, args, err := squirrel.Insert(Table("module")). Columns("id", "name", "last_connected_at"). Values(module.Id, module.Name, squirrel.Expr("unixepoch()")). Suffix(`on conflict (name) do update set last_connected_at = unixepoch()`). - Suffix("returning id"). ToSql() if err != nil { - return "", errors.WithMessage(err, "build query") + return errors.WithMessage(err, "build query") } - result := make(map[string]string) - err = r.db.SelectRow(ctx, &result, query, args...) + _, err = r.db.Exec(ctx, query, args...) if err != nil { - return "", errors.WithMessagef(err, "select: %s", query) + return errors.WithMessagef(err, "select: %s", query) } - return result["id"], nil + return nil } func (r Module) SetDisconnectedAtNow( diff --git a/service/module/service.go b/service/module/service.go index 722ea61..76527dc 100644 --- a/service/module/service.go +++ b/service/module/service.go @@ -23,7 +23,7 @@ const ( ) type Repo interface { - Upsert(ctx context.Context, module entity.Module) (string, error) + Upsert(ctx context.Context, module entity.Module) error SetDisconnectedAtNow( ctx context.Context, moduleId string, @@ -90,11 +90,12 @@ func NewService( func (s Service) OnConnect(ctx context.Context, conn *etp.Conn, moduleName string) error { s.logger.Info(ctx, "module connected", helpers.LogFields(conn)...) + moduleId := idByKey(moduleName) module := entity.Module{ - Id: uuid.NewString(), + Id: moduleId, Name: moduleName, } - moduleId, err := s.moduleRepo.Upsert(ctx, module) + err := s.moduleRepo.Upsert(ctx, module) if err != nil { return errors.WithMessage(err, "upsert module in store") } @@ -238,10 +239,8 @@ func (s Service) OnModuleConfigSchema( return errors.WithMessage(err, "get active config") } if config == nil { - hash := sha256.Sum256([]byte(moduleId)) - initialConfigId := hex.EncodeToString(hash[:]) initialConfig := entity.Config{ - Id: initialConfigId, + Id: idByKey(moduleId), Name: helpers.ModuleName(conn), ModuleId: moduleId, Data: data.Config, @@ -272,3 +271,8 @@ func (s Service) OnModuleConfigSchema( return nil } + +func idByKey(key string) string { + hash := sha256.Sum256([]byte(key)) + return hex.EncodeToString(hash[:]) +} From 0065c7c5a744c3be02b0a939c76a923dfed37652 Mon Sep 17 00:00:00 2001 From: d1slike Date: Fri, 24 May 2024 18:26:39 +0300 Subject: [PATCH 22/40] Add migration tool --- cmd/migrate/build.sh | 2 ++ cmd/migrate/main.go | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100755 cmd/migrate/build.sh diff --git a/cmd/migrate/build.sh b/cmd/migrate/build.sh new file mode 100755 index 0000000..2414980 --- /dev/null +++ b/cmd/migrate/build.sh @@ -0,0 +1,2 @@ +go install github.com/mitchellh/gox +gox -os="linux windows darwin" -arch="amd64" -output="build/{{.OS}}_{{.Arch}}/{{.Dir}}" diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index f012e70..12641cd 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -55,6 +55,7 @@ func main() { configSchemaRepo := repository.NewConfigSchema(dbClient) configHistoryRepo := repository.NewConfigHistory(dbClient) + logger.Info(ctx, "start modules migration") modules := make([]Module, 0) err = pgDb.Select(ctx, &modules, "select * from modules") if err != nil { @@ -73,8 +74,9 @@ func main() { } logger.Info(ctx, "modules migrated") + logger.Info(ctx, "start configs migration") configs := make([]Config, 0) - err = pgDb.Select(ctx, &configs, "select * from configs") + err = pgDb.Select(ctx, &configs, "select id, module_id, version, active, created_at, updated_at, data, name from configs") if err != nil { panic(errors.WithMessage(err, "err selecting configs")) } @@ -97,6 +99,7 @@ func main() { } logger.Info(ctx, "configs migrated") + logger.Info(ctx, "start config schemas migration") schemas := make([]ConfigSchema, 0) err = pgDb.Select(ctx, &schemas, "select * from config_schemas") if err != nil { @@ -118,6 +121,7 @@ func main() { } logger.Info(ctx, "config schemas migrated") + logger.Info(ctx, "start config history migration") configHistories := make([]ConfigHistory, 0) err = pgDb.Select(ctx, &configHistories, "select * from version_config") if err != nil { From 3a9e163c9e2acc7ea4994d4f58384ab0b414bdaa Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Sat, 7 Sep 2024 10:51:30 +0300 Subject: [PATCH 23/40] Bump rqlited --- Dockerfile | 4 +- go.mod | 69 ++++++------- go.sum | 152 ++++++++++++++-------------- service/rqlite/flags.go | 105 ++++++++++++++++++- service/rqlite/goose_store/store.go | 2 +- service/rqlite/main.go | 95 +++++++++++++---- 6 files changed, 292 insertions(+), 135 deletions(-) diff --git a/Dockerfile b/Dockerfile index ac056cc..8187316 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine3.19 as builder +FROM golang:1.22-alpine3.20 as builder WORKDIR /build ARG version ENV version_env=$version @@ -8,7 +8,7 @@ COPY . . RUN apk update && apk upgrade && apk add --no-cache gcc musl-dev RUN go build -ldflags="-X 'main.version=$version_env'" -o /main . -FROM alpine:3.19 +FROM alpine:3.20 WORKDIR /app ARG app_name ENV app_name_env=$app_name diff --git a/go.mod b/go.mod index 868ba78..624696a 100644 --- a/go.mod +++ b/go.mod @@ -6,17 +6,17 @@ require ( github.com/Masterminds/squirrel v1.5.4 github.com/google/uuid v1.6.0 github.com/pkg/errors v0.9.1 - github.com/pressly/goose/v3 v3.20.0 + github.com/pressly/goose/v3 v3.21.1 github.com/prometheus/client_golang v1.19.1 - github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 - github.com/rqlite/rqlite/v8 v8.23.4 + github.com/rqlite/gorqlite v0.0.0-20240808172217-12ae7d03ef19 + github.com/rqlite/rqlite/v8 v8.30.0 github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.17.1 - github.com/txix-open/etp/v3 v3.1.0 - github.com/txix-open/isp-kit v1.31.2 + github.com/txix-open/etp/v3 v3.2.0 + github.com/txix-open/isp-kit v1.38.1 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/sync v0.7.0 - google.golang.org/grpc v1.63.2 + golang.org/x/sync v0.8.0 + google.golang.org/grpc v1.65.0 ) require ( @@ -26,22 +26,23 @@ require ( github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/coder/websocket v1.8.12 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/fatih/color v1.16.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/getsentry/sentry-go v0.27.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/fatih/color v1.17.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.5 // indirect + github.com/getsentry/sentry-go v0.28.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect - github.com/hashicorp/raft v1.6.1 // indirect + github.com/hashicorp/raft v1.7.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -53,14 +54,15 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.53.0 // indirect - github.com/prometheus/procfs v0.14.0 // indirect - github.com/rqlite/go-sqlite3 v1.32.0 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/rqlite/go-sqlite3 v1.35.0 // indirect github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 // indirect github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd // indirect - github.com/sethvargo/go-retry v0.2.4 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/txix-open/bellows v1.2.0 // indirect @@ -69,27 +71,26 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - go.etcd.io/bbolt v1.3.9 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.51.0 // indirect - go.opentelemetry.io/otel v1.26.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect - go.opentelemetry.io/otel/metric v1.26.0 // indirect - go.opentelemetry.io/otel/sdk v1.26.0 // indirect - go.opentelemetry.io/otel/trace v1.26.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - nhooyr.io/websocket v1.8.11 // indirect ) replace ( diff --git a/go.sum b/go.sum index 1bc8b96..84e9b5f 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= +github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -30,14 +32,14 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= -github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= +github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= +github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= +github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -45,8 +47,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -55,8 +57,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA= -github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= +github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= +github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -71,8 +73,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 h1:CWyXh/jylQWp2dtiV33mY4iSSp6yf4lmn+c7/tN+ObI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0/go.mod h1:nCLIt0w3Ept2NwF8ThLmrppXsfT07oC8k0XNDxd8sVU= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -97,8 +99,8 @@ github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.6.1 h1:v/jm5fcYHvVkL0akByAp+IDdDSzCNCGhdO6VdB56HIM= -github.com/hashicorp/raft v1.6.1/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0= +github.com/hashicorp/raft v1.7.1 h1:ytxsNx4baHsRZrhUcbt3+79zc4ly8qm7pi0393pSchY= +github.com/hashicorp/raft v1.7.1/go.mod h1:hUeiEwQQR/Nk2iKDD0dkEhklSsu3jcAcqvPzPoZSAEM= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -146,6 +148,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= @@ -160,13 +164,11 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0= -github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA= +github.com/pressly/goose/v3 v3.21.1 h1:5SSAKKWej8LVVzNLuT6KIvP1eFDuPvxa+B6H0w78buQ= +github.com/pressly/goose/v3 v3.21.1/go.mod h1:sqthmzV8PitchEkjecFJII//l43dLOCzfWh8pHEe+vE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -176,31 +178,31 @@ github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= -github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s= -github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= -github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= -github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rqlite/go-sqlite3 v1.32.0 h1:bB0IUDX8bswSpAmxltWtu/la2DV/GAccjprOcUEdihY= -github.com/rqlite/go-sqlite3 v1.32.0/go.mod h1:R9H7CatgYBt3c+fSV/5yo2vLh4ZjCB0aMHdkv69fP4A= -github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 h1:gYUQqzapdN4PQF5j0zDFI9ANQVAVFoJivNp5bTZEZMo= -github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rqlite/go-sqlite3 v1.35.0 h1:NTG3mmwZly6Bhfe8oi7HNtI9jUDIs6XtXGZZoVa/eRA= +github.com/rqlite/go-sqlite3 v1.35.0/go.mod h1:R9H7CatgYBt3c+fSV/5yo2vLh4ZjCB0aMHdkv69fP4A= +github.com/rqlite/gorqlite v0.0.0-20240808172217-12ae7d03ef19 h1:uuWunw893WVwpSg4kNBuS6swgABwc+rwInVtwR5E3eM= +github.com/rqlite/gorqlite v0.0.0-20240808172217-12ae7d03ef19/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 h1:NZ62M+kT0JqhyFUMc8I4SMmfmD4NGJxhb2ePJQXjryc= github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48/go.mod h1:CRnsxgy5G8fAf5J+AM0yrsSdxXHKkIYOaq2sm+Q4DYc= -github.com/rqlite/rqlite/v8 v8.23.4 h1:DK6/7u0RkGhJvDTMQ+Ta3SB+2fd62v0RUq3aDNumWmM= -github.com/rqlite/rqlite/v8 v8.23.4/go.mod h1:TlIWBD2OvzfuPzuG7SWXVS/N9suOHfSYRNmulYKF3ds= +github.com/rqlite/rqlite/v8 v8.30.0 h1:93htyRh3NQzHEavk4/r2AJyA5w2GturOOn7X4CAfVeU= +github.com/rqlite/rqlite/v8 v8.30.0/go.mod h1:j7PkXMRmrCZh24FUBbHL5788UKxvgDG6s0na8WGXIvM= github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd h1:wW6BtayFoKaaDeIvXRE3SZVPOscSKlYD+X3bB749+zk= github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg= -github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= -github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -227,12 +229,12 @@ github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgjeY= github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o= -github.com/txix-open/etp/v3 v3.1.0 h1:EQdjm0x2K6Dr6W8oRvs/dSU4rnw+g9o1THahIxLBC3E= -github.com/txix-open/etp/v3 v3.1.0/go.mod h1:aXqeKBdu55VOtmzx+PMlDAcz6eTNBMt3vP3NkiE1qj0= +github.com/txix-open/etp/v3 v3.2.0 h1:EgTchT8VtCYV1iEo/y4jII3/NybrXiQDE0y6pmS350M= +github.com/txix-open/etp/v3 v3.2.0/go.mod h1:lgen0TJmiturgBhvlxH1/PQ7iLL/OebKbxPAQYkuKI0= github.com/txix-open/grmq v1.6.0 h1:V4QRb0sI2CMfdMaMsFGvGvUIDGad5vr3khNU7A3LlP8= github.com/txix-open/grmq v1.6.0/go.mod h1:s+NNNv42+32yLpjvl2tpqtyBbMB9e7kKFee3lfa/dyo= -github.com/txix-open/isp-kit v1.31.2 h1:qt6Om1wF+vpbIz01jQix9L20h9ftwn3/SasgO1RT9T0= -github.com/txix-open/isp-kit v1.31.2/go.mod h1:t6/NHdK5JvpIiQU4+NUjky4jLbo7Bz/4hzpoBxZt64U= +github.com/txix-open/isp-kit v1.38.1 h1:IYsiW2xc06//xQsWcHX9A1GXCGTXQRXbUqotcxa6eDY= +github.com/txix-open/isp-kit v1.38.1/go.mod h1:uVn5y71Rg8jlFjFK3+QCh9uw1O7mcK+1grCNFZRWhME= github.com/txix-open/jsonschema v1.2.0 h1:B8TrdSsPhDvYv67oi/LVqf7/+PuqFlMsNwUzGeHfl1s= github.com/txix-open/jsonschema v1.2.0/go.mod h1:l8YDZ1nvJrw6uxWowSVOxCV/ebiMJyapffW87ZEqH00= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= @@ -248,26 +250,26 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.51.0 h1:974XTyIwHI4nHa1+uSLxHtUnlJ2DiVtAJjk7fd07p/8= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.51.0/go.mod h1:ZvX/taFlN6TGaOOM6D42wrNwPKUV1nGO2FuUXkityBU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= -go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= -go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.53.0 h1:IVtyPth4Rs5P8wIf0mP2KVKFNTJ4paX9qQ4Hkh5gFdc= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.53.0/go.mod h1:ImRBLMJv177/pwiLZ7tU7HDGNdBv7rS0HQ99eN/zBl8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -277,8 +279,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -286,15 +288,15 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -312,8 +314,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= @@ -323,14 +325,14 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -359,5 +361,3 @@ modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= -nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= diff --git a/service/rqlite/flags.go b/service/rqlite/flags.go index 9f38ec4..425accf 100644 --- a/service/rqlite/flags.go +++ b/service/rqlite/flags.go @@ -35,12 +35,37 @@ const ( NodeX509KeyFlag = "node-key" ) +// StringSlice is a slice of strings which implments the flag.Value interface. +type StringSliceValue []string + +// String returns a string representation of the slice. +func (s *StringSliceValue) String() string { + return fmt.Sprintf("%v", *s) +} + +// Set sets the value of the slice. +func (s *StringSliceValue) Set(value string) error { + *s = strings.Split(value, ",") + var r []string + for _, v := range *s { + if v != "" { + r = append(r, v) + } + } + *s = r + return nil +} + // Config represents the configuration as set by command-line flags. // All variables will be set, unless explicit noted. type Config struct { // DataPath is path to node data. Always set. DataPath string + // ExtensionPaths is a comma-delimited list of path to SQLite extensions to be loaded. + // Each element may be a directory path, zipfile, or tar.gz file. May not be set. + ExtensionPaths StringSliceValue + // HTTPAddr is the bind network address for the HTTP Server. // It never includes a trailing HTTP or HTTPS. HTTPAddr string @@ -139,6 +164,9 @@ type Config struct { // AutoVacInterval sets the automatic VACUUM interval. Use 0s to disable. AutoVacInterval time.Duration + // AutoOptimizeInterval sets the automatic optimization interval. Use 0s to disable. + AutoOptimizeInterval time.Duration + // RaftLogLevel sets the minimum logging level for the Raft subsystem. RaftLogLevel string @@ -207,6 +235,9 @@ type Config struct { // MemProfile enables memory profiling. MemProfile string + + // TraceProfile enables trace profiling. + TraceProfile string } // Validate checks the configuration for internal consistency, and activates @@ -225,6 +256,19 @@ func (c *Config) Validate() error { return err } + err = c.CheckDirPaths() + if err != nil { + return err + } + + if len(c.ExtensionPaths) > 0 { + for _, p := range c.ExtensionPaths { + if !fileExists(p) { + return fmt.Errorf("extension path does not exist: %s", p) + } + } + } + if !bothUnsetSet(c.HTTPx509Cert, c.HTTPx509Key) { return fmt.Errorf("either both -%s and -%s must be set, or neither", HTTPx509CertFlag, HTTPx509KeyFlag) } @@ -408,8 +452,7 @@ func (c *Config) CheckFilePaths() error { if filePath == "" { continue } - _, err := os.Stat(filePath) - if os.IsNotExist(err) { + if !fileExists(filePath) { return fmt.Errorf("%s does not exist", filePath) } } @@ -417,6 +460,36 @@ func (c *Config) CheckFilePaths() error { return nil } +// CheckDirPaths checks that all directory paths in the config exist and are directories. +// Empty directory paths are ignored. +func (c *Config) CheckDirPaths() error { + v := reflect.ValueOf(c).Elem() + + // Iterate through the fields of the struct + for i := 0; i < v.NumField(); i++ { + field := v.Type().Field(i) + fieldValue := v.Field(i) + + if fieldValue.Kind() != reflect.String { + continue + } + + if tagValue, ok := field.Tag.Lookup("dirpath"); ok && tagValue == "true" { + dirPath := fieldValue.String() + if dirPath == "" { + continue + } + if !fileExists(dirPath) { + return fmt.Errorf("%s does not exist", dirPath) + } + if !isDir(dirPath) { + return fmt.Errorf("%s is not a directory", dirPath) + } + } + } + return nil +} + // BuildInfo is build information for display at command line. type BuildInfo struct { Version string @@ -427,12 +500,15 @@ type BuildInfo struct { // ParseFlags parses the command line, and returns the configuration. func ParseFlags(name, desc string, build *BuildInfo) (*Config, error) { - config := &Config{} + config := &Config{ + ExtensionPaths: StringSliceValue{}, + } showVersion := false fs := flag.NewFlagSet(name, flag.ContinueOnError) fs.StringVar(&config.NodeID, "node-id", "", "Unique ID for node. If not set, set to advertised Raft address") + fs.Var(&config.ExtensionPaths, "extensions-path", "Comma-delimited list of paths to directories, zipfiles, or tar.gz files containing SQLite extensions") fs.StringVar(&config.HTTPAddr, HTTPAddrFlag, "localhost:4001", "HTTP server bind address. To enable HTTPS, set X.509 certificate and key") fs.StringVar(&config.HTTPAdv, HTTPAdvAddrFlag, "", "Advertised HTTP address. If not set, same as HTTP server bind address") fs.StringVar(&config.HTTPAllowOrigin, "http-allow-origin", "", "Value to set for Access-Control-Allow-Origin HTTP header") @@ -464,6 +540,7 @@ func ParseFlags(name, desc string, build *BuildInfo) (*Config, error) { fs.BoolVar(&config.FKConstraints, "fk", false, "Enable SQLite foreign key constraints") fs.BoolVar(&showVersion, "version", false, "Show version information and exit") fs.DurationVar(&config.AutoVacInterval, "auto-vacuum-int", 0, "Period between automatic VACUUMs. It not set, not enabled") + fs.DurationVar(&config.AutoOptimizeInterval, "auto-optimize-int", mustParseDuration("24h"), `Period between automatic 'PRAGMA optimize'. Set to 0h to disable`) fs.BoolVar(&config.RaftNonVoter, "raft-non-voter", false, "Configure as non-voting node") fs.DurationVar(&config.RaftHeartbeatTimeout, "raft-timeout", time.Second, "Raft heartbeat timeout") fs.DurationVar(&config.RaftElectionTimeout, "raft-election-timeout", time.Second, "Raft election timeout") @@ -485,6 +562,7 @@ func ParseFlags(name, desc string, build *BuildInfo) (*Config, error) { fs.BoolVar(&config.WriteQueueTx, "write-queue-tx", false, "Use a transaction when processing a queued write") fs.StringVar(&config.CPUProfile, "cpu-profile", "", "Path to file for CPU profiling information") fs.StringVar(&config.MemProfile, "mem-profile", "", "Path to file for memory profiling information") + fs.StringVar(&config.TraceProfile, "trace-profile", "", "Path to file for trace profiling information") fs.Usage = func() { fmt.Fprintf(os.Stderr, "\n%s\n\n", desc) fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", name) @@ -527,3 +605,24 @@ func errorExit(code int, msg string) { func bothUnsetSet(a, b string) bool { return (a == "" && b == "") || (a != "" && b != "") } + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func isDir(path string) bool { + fi, err := os.Stat(path) + if err != nil { + return false + } + return fi.IsDir() +} + +func mustParseDuration(d string) time.Duration { + td, err := time.ParseDuration(d) + if err != nil { + panic(err) + } + return td +} diff --git a/service/rqlite/goose_store/store.go b/service/rqlite/goose_store/store.go index 6a3f56a..6d021da 100644 --- a/service/rqlite/goose_store/store.go +++ b/service/rqlite/goose_store/store.go @@ -69,7 +69,7 @@ func (s Store) GetMigration(ctx context.Context, db database.DBTxConn, version i } func (s Store) GetLatestVersion(ctx context.Context, db database.DBTxConn) (int64, error) { - return -1, errors.New("not implemented") + return -1, nil } func (s Store) ListMigrations(ctx context.Context, db database.DBTxConn) ([]*database.ListMigrationsResult, error) { diff --git a/service/rqlite/main.go b/service/rqlite/main.go index 03a9a33..a2898c1 100644 --- a/service/rqlite/main.go +++ b/service/rqlite/main.go @@ -6,19 +6,22 @@ import ( "context" "crypto/tls" "fmt" + "github.com/pkg/errors" "log" "net" "os" + "path/filepath" "runtime" "strings" "time" - "github.com/pkg/errors" "github.com/rqlite/rqlite/v8/auth" "github.com/rqlite/rqlite/v8/cluster" "github.com/rqlite/rqlite/v8/cmd" "github.com/rqlite/rqlite/v8/db" + "github.com/rqlite/rqlite/v8/extensions" httpd "github.com/rqlite/rqlite/v8/http" + "github.com/rqlite/rqlite/v8/rarchive" "github.com/rqlite/rqlite/v8/rtls" "github.com/rqlite/rqlite/v8/store" "github.com/rqlite/rqlite/v8/tcp" @@ -65,8 +68,8 @@ func main(ctx context.Context, r *Rqlite) error { r.localHttpAddr = cfg.HTTPAddr // Configure logging and pump out initial message. - log.Printf("%s starting, version %s, SQLite %s, commit %s, branch %s, compiler %s", name, cmd.Version, - db.DBVersion, cmd.Commit, cmd.Branch, runtime.Compiler) + log.Printf("%s starting, version %s, SQLite %s, commit %s, branch %s, compiler (toolchain) %s, compiler (command) %s", + name, cmd.Version, db.DBVersion, cmd.Commit, cmd.Branch, runtime.Compiler, cmd.CompilerCommand) log.Printf("%s, target architecture is %s, operating system target is %s", runtime.Version(), runtime.GOARCH, runtime.GOOS) log.Printf("launch command: %s", strings.Join(os.Args, " ")) @@ -83,7 +86,6 @@ func main(ctx context.Context, r *Rqlite) error { // Raft internode layer raftLn := mux.Listen(cluster.MuxRaftHeader) - log.Printf("Raft TCP mux Listener registered with byte header %d", cluster.MuxRaftHeader) raftDialer, err := cluster.CreateRaftDialer(cfg.NodeX509Cert, cfg.NodeX509Key, cfg.NodeX509CACert, cfg.NodeVerifyServerName, cfg.NoNodeVerify) if err != nil { @@ -91,8 +93,18 @@ func main(ctx context.Context, r *Rqlite) error { } raftTn := tcp.NewLayer(raftLn, raftDialer) + // Create extension store. + extensionsStore, err := createExtensionsStore(cfg) + if err != nil { + log.Fatalf("failed to create extensions store: %s", err.Error()) + } + extensionsPaths, err := extensionsStore.List() + if err != nil { + log.Fatalf("failed to list extensions: %s", err.Error()) + } + // Create the store. - str, err := createStore(cfg, raftTn) + str, err := createStore(cfg, raftTn, extensionsPaths) if err != nil { log.Fatalf("failed to create store: %s", err.Error()) } @@ -110,7 +122,6 @@ func main(ctx context.Context, r *Rqlite) error { if err != nil { log.Fatalf("failed to create cluster service: %s", err.Error()) } - log.Printf("cluster TCP mux Listener registered with byte header %d", cluster.MuxClusterHeader) // Create the HTTP service. // @@ -132,8 +143,18 @@ func main(ctx context.Context, r *Rqlite) error { } // Register remaining status providers. - httpServ.RegisterStatus("cluster", clstrServ) - httpServ.RegisterStatus("network", tcp.NetworkReporter{}) + if err := httpServ.RegisterStatus("cluster", clstrServ); err != nil { + log.Fatalf("failed to register cluster status provider: %s", err.Error()) + } + if err := httpServ.RegisterStatus("network", tcp.NetworkReporter{}); err != nil { + log.Fatalf("failed to register network status provider: %s", err.Error()) + } + if err := httpServ.RegisterStatus("mux", mux); err != nil { + log.Fatalf("failed to register mux status provider: %s", err.Error()) + } + if err := httpServ.RegisterStatus("extensions", extensionsStore); err != nil { + log.Fatalf("failed to register extensions status provider: %s", err.Error()) + } // Create the cluster! nodes, err := str.Nodes() @@ -152,9 +173,10 @@ func main(ctx context.Context, r *Rqlite) error { // Block until done. <-mainCtx.Done() - // Stop the HTTP server first, so clients get notification as soon as + // Stop the HTTP server and other network access first so clients get notification as soon as // possible that the node is going away. httpServ.Close() + clstrServ.Close() if cfg.RaftClusterRemoveOnShutdown { remover := cluster.NewRemover(clstrClient, 5*time.Second, str) @@ -174,19 +196,52 @@ func main(ctx context.Context, r *Rqlite) error { // Perform a stepdown, ignore any errors. str.Stepdown(true) } + muxLn.Close() if err := str.Close(true); err != nil { log.Printf("failed to close store: %s", err.Error()) } - clstrServ.Close() - muxLn.Close() + + log.Println("rqlite server stopped") return nil } -func createStore(cfg *Config, ln *tcp.Layer) (*store.Store, error) { +func createExtensionsStore(cfg *Config) (*extensions.Store, error) { + str, err := extensions.NewStore(filepath.Join(cfg.DataPath, "extensions")) + if err != nil { + log.Fatalf("failed to create extension store: %s", err.Error()) + } + + if len(cfg.ExtensionPaths) > 0 { + for _, path := range cfg.ExtensionPaths { + if isDir(path) { + if err := str.LoadFromDir(path); err != nil { + log.Fatalf("failed to load extensions from directory: %s", err.Error()) + } + } else if rarchive.IsZipFile(path) { + if err := str.LoadFromZip(path); err != nil { + log.Fatalf("failed to load extensions from zip file: %s", err.Error()) + } + } else if rarchive.IsTarGzipFile(path) { + if err := str.LoadFromTarGzip(path); err != nil { + log.Fatalf("failed to load extensions from tar.gz file: %s", err.Error()) + } + } else { + if err := str.LoadFromFile(path); err != nil { + log.Fatalf("failed to load extension from file: %s", err.Error()) + } + } + } + } + + return str, nil +} + +func createStore(cfg *Config, ln *tcp.Layer, extensions []string) (*store.Store, error) { dbConf := store.NewDBConfig() dbConf.OnDiskPath = cfg.OnDiskPath dbConf.FKConstraints = cfg.FKConstraints + dbConf.Extensions = extensions str := store.New(ln, &store.Config{ DBConf: dbConf, @@ -208,6 +263,7 @@ func createStore(cfg *Config, ln *tcp.Layer) (*store.Store, error) { str.ReapTimeout = cfg.RaftReapNodeTimeout str.ReapReadOnlyTimeout = cfg.RaftReapReadOnlyNodeTimeout str.AutoVacInterval = cfg.AutoVacInterval + str.AutoOptimizeInterval = cfg.AutoOptimizeInterval if store.IsNewNode(cfg.DataPath) { log.Printf("no preexisting node state detected in %s, node may be bootstrapping", cfg.DataPath) @@ -230,14 +286,15 @@ func startHTTPService(cfg *Config, str *store.Store, cltr *cluster.Client, credS s.DefaultQueueBatchSz = cfg.WriteQueueBatchSz s.DefaultQueueTimeout = cfg.WriteQueueTimeout s.DefaultQueueTx = cfg.WriteQueueTx - s.AllowOrigin = cfg.HTTPAllowOrigin s.BuildInfo = map[string]interface{}{ - "commit": cmd.Commit, - "branch": cmd.Branch, - "version": cmd.Version, - "compiler": runtime.Compiler, - "build_time": cmd.Buildtime, - } + "commit": cmd.Commit, + "branch": cmd.Branch, + "version": cmd.Version, + "compiler_toolchain": runtime.Compiler, + "compiler_command": cmd.CompilerCommand, + "build_time": cmd.Buildtime, + } + s.SetAllowOrigin(cfg.HTTPAllowOrigin) return s, s.Start() } From 38c4295d0b55386068e4f83682468b56632b5750 Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Sun, 8 Sep 2024 09:36:16 +0300 Subject: [PATCH 24/40] Use match elements --- cmd/migrate/go.mod | 18 +++++++++--------- cmd/migrate/go.sum | 9 +++++++++ tests/acceptance_test.go | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index 7d214ce..859013f 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -8,8 +8,8 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/mattn/go-sqlite3 v1.14.22 github.com/pkg/errors v0.9.1 - github.com/pressly/goose/v3 v3.20.0 - github.com/txix-open/isp-kit v1.31.2 + github.com/pressly/goose/v3 v3.21.1 + github.com/txix-open/isp-kit v1.38.1 isp-config-service v0.0.0-00010101000000-000000000000 ) @@ -17,8 +17,8 @@ require ( github.com/Masterminds/squirrel v1.5.4 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect - github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.6.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect @@ -28,17 +28,17 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/sethvargo/go-retry v0.2.4 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/txix-open/bellows v1.2.0 // indirect - github.com/txix-open/etp/v3 v3.1.0 // indirect + github.com/txix-open/etp/v3 v3.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/text v0.17.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect nhooyr.io/websocket v1.8.11 // indirect diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index f8d1a45..6506632 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -32,8 +32,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= @@ -74,12 +76,14 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0= github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA= +github.com/pressly/goose/v3 v3.21.1/go.mod h1:sqthmzV8PitchEkjecFJII//l43dLOCzfWh8pHEe+vE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -103,8 +107,10 @@ github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgj github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o= github.com/txix-open/etp/v3 v3.1.0 h1:EQdjm0x2K6Dr6W8oRvs/dSU4rnw+g9o1THahIxLBC3E= github.com/txix-open/etp/v3 v3.1.0/go.mod h1:aXqeKBdu55VOtmzx+PMlDAcz6eTNBMt3vP3NkiE1qj0= +github.com/txix-open/etp/v3 v3.2.0/go.mod h1:lgen0TJmiturgBhvlxH1/PQ7iLL/OebKbxPAQYkuKI0= github.com/txix-open/isp-kit v1.31.2 h1:qt6Om1wF+vpbIz01jQix9L20h9ftwn3/SasgO1RT9T0= github.com/txix-open/isp-kit v1.31.2/go.mod h1:t6/NHdK5JvpIiQU4+NUjky4jLbo7Bz/4hzpoBxZt64U= +github.com/txix-open/isp-kit v1.38.1/go.mod h1:uVn5y71Rg8jlFjFK3+QCh9uw1O7mcK+1grCNFZRWhME= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -115,14 +121,17 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/tests/acceptance_test.go b/tests/acceptance_test.go index df7dcf4..b7925fb 100644 --- a/tests/acceptance_test.go +++ b/tests/acceptance_test.go @@ -102,7 +102,7 @@ func TestAcceptance(t *testing.T) { require.EqualValues([]byte("{}"), eventHandler.ReceivedConfigs()[0]) require.Len(eventHandler.ReceivedHosts(), 2) - require.EqualValues([]string{"10.2.9.1:9999", "10.2.9.2:9999"}, eventHandler.ReceivedHosts()[0]) + require.ElementsMatch([]string{"10.2.9.1:9999", "10.2.9.2:9999"}, eventHandler.ReceivedHosts()[0]) require.EqualValues([]string{"10.2.9.1:9999"}, eventHandler.ReceivedHosts()[1]) require.Len(eventHandler.ReceivedRoutes(), 3) From e1547dbb94730a0a29e6b429aae0253b106e5ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B8=D0=BD=D0=B5=D0=BB=D1=8C=D0=BD=D0=B8=D0=BA=20?= =?UTF-8?q?=D0=A2=D0=B8=D0=BC=D1=83=D1=80=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 2 Oct 2024 10:02:25 +0300 Subject: [PATCH 25/40] MSP-4182 Add autoincrement to isp_config_service__event id field --- ...20240503120010_add_event_autoincrement.sql | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 migrations/20240503120010_add_event_autoincrement.sql diff --git a/migrations/20240503120010_add_event_autoincrement.sql b/migrations/20240503120010_add_event_autoincrement.sql new file mode 100644 index 0000000..d660eed --- /dev/null +++ b/migrations/20240503120010_add_event_autoincrement.sql @@ -0,0 +1,31 @@ +-- +goose Up +-- +goose NO TRANSACTION +CREATE TABLE isp_config_service__event_autoincrement ( + id integer primary key autoincrement, + payload blob not null, + created_at integer not null default (unixepoch()) +); + +INSERT INTO + isp_config_service__event_autoincrement (id, payload, created_at) +SELECT id, payload, created_at +FROM isp_config_service__event; +DROP TABLE isp_config_service__event; + +ALTER TABLE isp_config_service__event_autoincrement RENAME TO isp_config_service__event; +-- +goose Down +CREATE TABLE isp_config_service__event_without_autoincrement ( + id integer primary key, + payload blob not null, + created_at integer not null default (unixepoch()) +); + +INSERT INTO + isp_config_service__event_without_autoincrement (id, payload, created_at) +SELECT id, payload, created_at +FROM isp_config_service__event; + +DROP TABLE isp_config_service__event; + +ALTER TABLE isp_config_service__event_without_autoincrement +RENAME TO isp_config_service__event; \ No newline at end of file From 99c0e1b17d72be5b0be73ba63d54a4be6ac6eabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B8=D0=BD=D0=B5=D0=BB=D1=8C=D0=BD=D0=B8=D0=BA=20?= =?UTF-8?q?=D0=A2=D0=B8=D0=BC=D1=83=D1=80=20=D0=A0=D0=BE=D0=BC=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Wed, 2 Oct 2024 10:07:57 +0300 Subject: [PATCH 26/40] MSP-4182 Bump version to 3.0.1 --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 4a36342..cb2b00e 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -3.0.0 +3.0.1 From e52c0f760cea08b138efd83fa756066c150b3fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D1=83=D1=87=D0=BA=D0=BE=D0=B2=20=D0=90=D0=BD=D0=B4?= =?UTF-8?q?=D1=80=D0=B5=D0=B9=20=D0=9F=D0=B0=D0=B2=D0=BB=D0=BE=D0=B2=D0=B8?= =?UTF-8?q?=D1=87?= Date: Mon, 14 Oct 2024 16:51:05 +0300 Subject: [PATCH 27/40] MSP-4227 fix module name --- domain/response.go | 1 + service/api/module.go | 1 + 2 files changed, 2 insertions(+) diff --git a/domain/response.go b/domain/response.go index c8ffe37..80db8bf 100644 --- a/domain/response.go +++ b/domain/response.go @@ -22,6 +22,7 @@ type ModuleInfo struct { } type Connection struct { + ModuleName string LibVersion string Version string Address Address diff --git a/service/api/module.go b/service/api/module.go index b5966cb..8e5c260 100644 --- a/service/api/module.go +++ b/service/api/module.go @@ -150,6 +150,7 @@ func (s Module) backendToDto(backend entity.Backend) (domain.Connection, error) }) conn := domain.Connection{ + ModuleName: backend.ModuleName, LibVersion: backend.LibVersion, Version: backend.Version, Address: domain.Address{ From 62815a76910905d48e2a9b5f8cf4b299128e463c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D1=83=D1=87=D0=BA=D0=BE=D0=B2=20=D0=90=D0=BD=D0=B4?= =?UTF-8?q?=D1=80=D0=B5=D0=B9=20=D0=9F=D0=B0=D0=B2=D0=BB=D0=BE=D0=B2=D0=B8?= =?UTF-8?q?=D1=87?= Date: Mon, 14 Oct 2024 16:51:19 +0300 Subject: [PATCH 28/40] MSP-4227 Bump version to 3.0.2 --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index cb2b00e..b502146 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -3.0.1 +3.0.2 From c32c9d5eba28c79575d5aeb46f20706ffeef640c Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Mon, 28 Oct 2024 20:24:48 +0300 Subject: [PATCH 29/40] Implement auth --- Dockerfile | 18 +++++- assembly/locator.go | 19 +++++- cmd/migrate/build.sh | 2 +- cmd/migrate/config.yml | 12 ++++ cmd/migrate/go.mod | 16 ++--- cmd/migrate/go.sum | 71 ++++++++++------------ cmd/migrate/main.go | 57 ++++++++++++++++-- conf/config.yml | 13 +++- conf/local.go | 6 +- go.mod | 47 ++++++++------- go.sum | 110 +++++++++++++++++----------------- main.go | 7 ++- routes/routes.go | 9 ++- service/rqlite/credentials.go | 23 +++++++ service/rqlite/db/db.go | 3 +- service/rqlite/main.go | 8 ++- service/rqlite/runner.go | 33 ++++++++-- service/startup/service.go | 66 +++++++++++++------- tests/acceptance_test.go | 5 +- 19 files changed, 349 insertions(+), 176 deletions(-) create mode 100644 cmd/migrate/config.yml create mode 100644 service/rqlite/credentials.go diff --git a/Dockerfile b/Dockerfile index 8187316..e6b38dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine3.20 as builder +FROM golang:1.23-alpine3.20 as builder WORKDIR /build ARG version ENV version_env=$version @@ -9,6 +9,22 @@ RUN apk update && apk upgrade && apk add --no-cache gcc musl-dev RUN go build -ldflags="-X 'main.version=$version_env'" -o /main . FROM alpine:3.20 + +RUN apk add --no-cache tzdata +RUN cp /usr/share/zoneinfo/Europe/Moscow /etc/localtime +RUN echo "Europe/Moscow" > /etc/timezone + +ARG UID=10001 +RUN adduser \ + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + appuser +USER appuser + WORKDIR /app ARG app_name ENV app_name_env=$app_name diff --git a/assembly/locator.go b/assembly/locator.go index 925430c..3eb5a34 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -1,7 +1,10 @@ package assembly import ( + "isp-config-service/conf" "net/http" + "net/http/httputil" + "net/url" "time" "github.com/txix-open/etp/v3" @@ -27,14 +30,21 @@ const ( eventTtl = 60 * time.Second ) +type LocalConfig struct { + Local conf.Local + RqliteAddress string +} + type Locator struct { db db.DB + cfg LocalConfig logger log.Logger } -func NewLocator(logger log.Logger, db db.DB) Locator { +func NewLocator(logger log.Logger, db db.DB, cfg LocalConfig) Locator { return Locator{ db: db, + cfg: cfg, logger: logger, } } @@ -114,7 +124,12 @@ func (l Locator) Config() Config { routes.BindEtp(etpSrv, controllers, l.logger) - httpMux := routes.HttpHandler(etpSrv) + rqliteUrl, err := url.Parse(l.cfg.RqliteAddress) + if err != nil { + panic(err) + } + rqliteProxy := httputil.NewSingleHostReverseProxy(rqliteUrl) + httpMux := routes.HttpHandler(etpSrv, l.cfg.Local, rqliteProxy) eventHandler := event.NewHandler(subscriptionService, l.logger) handleEventJob := event.NewWorker(eventRepo, eventHandler, l.logger) diff --git a/cmd/migrate/build.sh b/cmd/migrate/build.sh index 2414980..1936d52 100755 --- a/cmd/migrate/build.sh +++ b/cmd/migrate/build.sh @@ -1,2 +1,2 @@ -go install github.com/mitchellh/gox +go install github.com/mitchellh/gox@latest gox -os="linux windows darwin" -arch="amd64" -output="build/{{.OS}}_{{.Arch}}/{{.Dir}}" diff --git a/cmd/migrate/config.yml b/cmd/migrate/config.yml new file mode 100644 index 0000000..6f7e5f7 --- /dev/null +++ b/cmd/migrate/config.yml @@ -0,0 +1,12 @@ +db: + host: 127.0.0.1 + port: 5432 + database: test + username: test + password: test + schema: config_service + +newConfigService: + address: http://127.0.0.1:9002/ + username: default + password: test \ No newline at end of file diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index 859013f..f5f700e 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -1,6 +1,6 @@ module isp-config-service/migrate -go 1.22 +go 1.23 replace isp-config-service => ../.. @@ -8,18 +8,19 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/mattn/go-sqlite3 v1.14.22 github.com/pkg/errors v0.9.1 - github.com/pressly/goose/v3 v3.21.1 - github.com/txix-open/isp-kit v1.38.1 + github.com/pressly/goose/v3 v3.22.1 + github.com/txix-open/isp-kit v1.39.0 isp-config-service v0.0.0-00010101000000-000000000000 ) require ( github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/coder/websocket v1.8.12 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -36,10 +37,9 @@ require ( github.com/txix-open/etp/v3 v3.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.26.0 // indirect + golang.org/x/crypto v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/text v0.19.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - nhooyr.io/websocket v1.8.11 // indirect ) diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 6506632..6f185d2 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -4,20 +4,22 @@ github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8 github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= +github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= +github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-resty/resty/v2 v2.12.0 h1:rsVL8P90LFvkUYq/V5BTVe203WfRIU4gvcf+yfzJzGA= -github.com/go-resty/resty/v2 v2.12.0/go.mod h1:o0yGPrkS3lOe1+eFajk6kBW8ScXzwU3hD69/gt2yB/0= +github.com/go-resty/resty/v2 v2.15.3 h1:bqff+hcqAflpiF591hhJzNdkRsFhlB96CYfBwSFvql8= +github.com/go-resty/resty/v2 v2.15.3/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -30,14 +32,12 @@ github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSAS github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= -github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -74,15 +74,13 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose/v3 v3.20.0 h1:uPJdOxF/Ipj7ABVNOAMJXSxwFXZGwMGHNqjC8e61VA0= -github.com/pressly/goose/v3 v3.20.0/go.mod h1:BRfF2GcG4FTG12QfdBVy3q1yveaf4ckL9vWwEcIO3lA= -github.com/pressly/goose/v3 v3.21.1/go.mod h1:sqthmzV8PitchEkjecFJII//l43dLOCzfWh8pHEe+vE= +github.com/pressly/goose/v3 v3.22.1 h1:2zICEfr1O3yTP9BRZMGPj7qFxQ+ik6yeo+z1LMuioLc= +github.com/pressly/goose/v3 v3.22.1/go.mod h1:xtMpbstWyCpyH+0cxLTMCENWBG+0CSxvTsXhW95d5eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= -github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -105,12 +103,10 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgjeY= github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o= -github.com/txix-open/etp/v3 v3.1.0 h1:EQdjm0x2K6Dr6W8oRvs/dSU4rnw+g9o1THahIxLBC3E= -github.com/txix-open/etp/v3 v3.1.0/go.mod h1:aXqeKBdu55VOtmzx+PMlDAcz6eTNBMt3vP3NkiE1qj0= +github.com/txix-open/etp/v3 v3.2.0 h1:EgTchT8VtCYV1iEo/y4jII3/NybrXiQDE0y6pmS350M= github.com/txix-open/etp/v3 v3.2.0/go.mod h1:lgen0TJmiturgBhvlxH1/PQ7iLL/OebKbxPAQYkuKI0= -github.com/txix-open/isp-kit v1.31.2 h1:qt6Om1wF+vpbIz01jQix9L20h9ftwn3/SasgO1RT9T0= -github.com/txix-open/isp-kit v1.31.2/go.mod h1:t6/NHdK5JvpIiQU4+NUjky4jLbo7Bz/4hzpoBxZt64U= -github.com/txix-open/isp-kit v1.38.1/go.mod h1:uVn5y71Rg8jlFjFK3+QCh9uw1O7mcK+1grCNFZRWhME= +github.com/txix-open/isp-kit v1.39.0 h1:NBV2oJJefH/qt2pcPrqEmYccJedjPaVCwr6Z955C+pg= +github.com/txix-open/isp-kit v1.39.0/go.mod h1:R4vwujGjMXLQir8MDDCfsDQa0CWGuZBSMakt5qcybgM= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -119,19 +115,16 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -142,17 +135,13 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= -modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= -modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= -modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.33.0 h1:WWkA/T2G17okiLGgKAj4/RMIvgyMT19yQ038160IeYk= +modernc.org/sqlite v1.33.0/go.mod h1:9uQ9hF/pCZoYZK73D/ud5Z7cIRIILSZI8NdIemVMTX8= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= -nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index 12641cd..e9dba18 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -2,6 +2,12 @@ package main import ( "fmt" + "github.com/txix-open/isp-kit/config" + "github.com/txix-open/isp-kit/dbx" + "github.com/txix-open/isp-kit/http/httpcli" + "github.com/txix-open/isp-kit/http/httpclix" + "github.com/txix-open/isp-kit/log" + "github.com/txix-open/isp-kit/validator" "os" "time" @@ -17,19 +23,33 @@ import ( "isp-config-service/repository" ) -func main() { - if len(os.Args) == 1 { - fmt.Println("usage: migrate postgres://username:password@localhost:5432/database_name?search_path=config_service") - os.Exit(1) +type Cfg struct { + Db dbx.Config + NewConfigService struct { + Address string + Username string + Password string } - app, err := app.New() +} + +func main() { + app, err := app.New(app.WithConfigOptions( + config.WithValidator(validator.Default), + config.WithExtraSource(config.NewYamlConfig("config.yml")), + )) if err != nil { panic(err) } logger := app.Logger() + logger.SetLevel(log.DebugLevel) ctx := app.Context() + cfg := Cfg{} + err = app.Config().Read(&cfg) + if err != nil { + panic(errors.WithMessage(err, "read config")) + } - pgDb, err := db.Open(ctx, os.Args[1]) + pgDb, err := dbx.Open(ctx, cfg.Db, dbx.WithQueryTracer(dbx.NewLogTracer(logger))) if err != nil { panic(err) } @@ -143,5 +163,30 @@ func main() { } logger.Info(ctx, "config history migrated") + logger.Info(ctx, "close sqlite db") + _ = dbClient.Close() + + logger.Info(ctx, "loading db to isp-config-service") + sqliteFile, err := os.ReadFile(fileName) + if err != nil { + panic(errors.WithMessage(err, "err opening sqlite file")) + } + + cli := httpcli.New(httpcli.WithMiddlewares(httpclix.Log(logger))) + cli.GlobalRequestConfig().BaseUrl = cfg.NewConfigService.Address + cli.GlobalRequestConfig().BasicAuth = &httpcli.BasicAuth{ + Username: cfg.NewConfigService.Username, + Password: cfg.NewConfigService.Password, + } + ctx = httpclix.LogConfigToContext(ctx, false, true) + err = cli.Post("/db/load"). + Header("Content-Type", "application/octet-stream"). + RequestBody(sqliteFile). + StatusCodeToError(). + DoWithoutResponse(ctx) + if err != nil { + panic(errors.WithMessage(err, "load sqlite file")) + } + logger.Info(ctx, "done!") } diff --git a/conf/config.yml b/conf/config.yml index b9331a8..1472ba5 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -2,7 +2,7 @@ configServiceAddress: ip: 127.0.0.1 port: 9001 grpcOuterAddress: - ip: 127.0.0.1 + ip: isp-config-service port: 9002 grpcInnerAddress: ip: 0.0.0.0 @@ -13,6 +13,15 @@ infraServerPort: 9006 logLevel: debug +maintenanceMode: false + +credentials: + - username: default + password: 38kPk0upqI5j + perms: ["all"] + +internalClientCredential: default + rqlite: DataPath: "data" NodeID: 1 @@ -21,4 +30,4 @@ rqlite: JoinAddrs: "127.0.0.1:9008" FKConstraints: true AutoVacInterval: 12h - + JoinAs: default diff --git a/conf/local.go b/conf/local.go index e73d137..3c13836 100644 --- a/conf/local.go +++ b/conf/local.go @@ -1,9 +1,13 @@ package conf import ( + "github.com/rqlite/rqlite/v8/auth" "github.com/txix-open/isp-kit/bootstrap" ) type Local struct { - ConfigServiceAddress bootstrap.ConfigServiceAddr + ConfigServiceAddress bootstrap.ConfigServiceAddr + MaintenanceMode bool + Credentials []auth.Credential + InternalClientCredential string `validate:"required"` } diff --git a/go.mod b/go.mod index 624696a..02d0868 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,22 @@ module isp-config-service -go 1.22 +go 1.23 require ( github.com/Masterminds/squirrel v1.5.4 github.com/google/uuid v1.6.0 github.com/pkg/errors v0.9.1 - github.com/pressly/goose/v3 v3.21.1 - github.com/prometheus/client_golang v1.19.1 + github.com/pressly/goose/v3 v3.22.1 + github.com/prometheus/client_golang v1.20.5 github.com/rqlite/gorqlite v0.0.0-20240808172217-12ae7d03ef19 github.com/rqlite/rqlite/v8 v8.30.0 github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.17.1 github.com/txix-open/etp/v3 v3.2.0 - github.com/txix-open/isp-kit v1.38.1 + github.com/txix-open/isp-kit v1.39.0 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.8.0 - google.golang.org/grpc v1.65.0 + google.golang.org/grpc v1.67.1 ) require ( @@ -29,13 +29,13 @@ require ( github.com/coder/websocket v1.8.12 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fatih/color v1.17.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.5 // indirect - github.com/getsentry/sentry-go v0.28.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.6 // indirect + github.com/getsentry/sentry-go v0.29.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect @@ -44,6 +44,7 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/raft v1.7.1 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -57,7 +58,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rqlite/go-sqlite3 v1.35.0 // indirect github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 // indirect @@ -72,23 +73,23 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.etcd.io/bbolt v1.3.10 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.53.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.26.0 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.17.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/protobuf v1.34.2 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 84e9b5f..9660a44 100644 --- a/go.sum +++ b/go.sum @@ -36,10 +36,10 @@ github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= -github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= -github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= -github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= +github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= +github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= +github.com/getsentry/sentry-go v0.29.1 h1:DyZuChN8Hz3ARxGVV8ePaNXh1dQ7d76AiB117xcREwA= +github.com/getsentry/sentry-go v0.29.1/go.mod h1:x3AtIzN01d6SiWkderzaH28Tm0lgkafpJ5Bm3li39O0= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -57,8 +57,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= -github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= +github.com/go-resty/resty/v2 v2.15.3 h1:bqff+hcqAflpiF591hhJzNdkRsFhlB96CYfBwSFvql8= +github.com/go-resty/resty/v2 v2.15.3/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -73,8 +73,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 h1:CWyXh/jylQWp2dtiV33mY4iSSp6yf4lmn+c7/tN+ObI= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0/go.mod h1:nCLIt0w3Ept2NwF8ThLmrppXsfT07oC8k0XNDxd8sVU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -110,6 +110,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -119,6 +121,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -164,13 +168,13 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pressly/goose/v3 v3.21.1 h1:5SSAKKWej8LVVzNLuT6KIvP1eFDuPvxa+B6H0w78buQ= -github.com/pressly/goose/v3 v3.21.1/go.mod h1:sqthmzV8PitchEkjecFJII//l43dLOCzfWh8pHEe+vE= +github.com/pressly/goose/v3 v3.22.1 h1:2zICEfr1O3yTP9BRZMGPj7qFxQ+ik6yeo+z1LMuioLc= +github.com/pressly/goose/v3 v3.22.1/go.mod h1:xtMpbstWyCpyH+0cxLTMCENWBG+0CSxvTsXhW95d5eo= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -178,8 +182,8 @@ github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= @@ -189,8 +193,8 @@ github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzuk github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rqlite/go-sqlite3 v1.35.0 h1:NTG3mmwZly6Bhfe8oi7HNtI9jUDIs6XtXGZZoVa/eRA= github.com/rqlite/go-sqlite3 v1.35.0/go.mod h1:R9H7CatgYBt3c+fSV/5yo2vLh4ZjCB0aMHdkv69fP4A= github.com/rqlite/gorqlite v0.0.0-20240808172217-12ae7d03ef19 h1:uuWunw893WVwpSg4kNBuS6swgABwc+rwInVtwR5E3eM= @@ -233,8 +237,8 @@ github.com/txix-open/etp/v3 v3.2.0 h1:EgTchT8VtCYV1iEo/y4jII3/NybrXiQDE0y6pmS350 github.com/txix-open/etp/v3 v3.2.0/go.mod h1:lgen0TJmiturgBhvlxH1/PQ7iLL/OebKbxPAQYkuKI0= github.com/txix-open/grmq v1.6.0 h1:V4QRb0sI2CMfdMaMsFGvGvUIDGad5vr3khNU7A3LlP8= github.com/txix-open/grmq v1.6.0/go.mod h1:s+NNNv42+32yLpjvl2tpqtyBbMB9e7kKFee3lfa/dyo= -github.com/txix-open/isp-kit v1.38.1 h1:IYsiW2xc06//xQsWcHX9A1GXCGTXQRXbUqotcxa6eDY= -github.com/txix-open/isp-kit v1.38.1/go.mod h1:uVn5y71Rg8jlFjFK3+QCh9uw1O7mcK+1grCNFZRWhME= +github.com/txix-open/isp-kit v1.39.0 h1:NBV2oJJefH/qt2pcPrqEmYccJedjPaVCwr6Z955C+pg= +github.com/txix-open/isp-kit v1.39.0/go.mod h1:R4vwujGjMXLQir8MDDCfsDQa0CWGuZBSMakt5qcybgM= github.com/txix-open/jsonschema v1.2.0 h1:B8TrdSsPhDvYv67oi/LVqf7/+PuqFlMsNwUzGeHfl1s= github.com/txix-open/jsonschema v1.2.0/go.mod h1:l8YDZ1nvJrw6uxWowSVOxCV/ebiMJyapffW87ZEqH00= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= @@ -252,22 +256,22 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.53.0 h1:IVtyPth4Rs5P8wIf0mP2KVKFNTJ4paX9qQ4Hkh5gFdc= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.53.0/go.mod h1:ImRBLMJv177/pwiLZ7tU7HDGNdBv7rS0HQ99eN/zBl8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -279,8 +283,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -288,8 +292,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -314,8 +318,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= @@ -325,14 +329,14 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 h1:2oV8dfuIkM1Ti7DwXc0BJfnwr9csz4TDXI9EmiI+Rbw= +google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38/go.mod h1:vuAjtvlwkDKF6L1GQ0SokiRLCGFfeBUXWr/aFFkHACc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -349,14 +353,12 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= -modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= -modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/sqlite v1.29.6 h1:0lOXGrycJPptfHDuohfYgNqoe4hu+gYuN/pKgY5XjS4= -modernc.org/sqlite v1.29.6/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/sqlite v1.33.0 h1:WWkA/T2G17okiLGgKAj4/RMIvgyMT19yQ038160IeYk= +modernc.org/sqlite v1.33.0/go.mod h1:9uQ9hF/pCZoYZK73D/ud5Z7cIRIILSZI8NdIemVMTX8= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/main.go b/main.go index a04f632..a176b1e 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,10 @@ func main() { app := boot.App logger := app.Logger() - startup := startup.New(boot) + startup, err := startup.New(boot) + if err != nil { + boot.Fatal(err) + } app.AddRunners(startup) app.AddClosers(startup.Closers()...) @@ -38,7 +41,7 @@ func main() { logger.Info(app.Context(), "shutdown completed") }) - err := app.Run() + err = app.Run() if err != nil { boot.Fatal(err) } diff --git a/routes/routes.go b/routes/routes.go index c631c0e..2ccbc16 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -1,6 +1,7 @@ package routes import ( + "isp-config-service/conf" "net/http" "github.com/txix-open/etp/v3" @@ -51,9 +52,13 @@ func BindEtp(etpSrv *etp.Server, c Controllers, logger log.Logger) { etpSrv.On(cluster.ModuleReady, onModuleReady) } -func HttpHandler(etpSrv *etp.Server) http.Handler { +func HttpHandler(etpSrv *etp.Server, conf conf.Local, rqliteProxy http.Handler) http.Handler { httpMux := http.NewServeMux() - httpMux.Handle("/isp-etp/", etpSrv) + if conf.MaintenanceMode { + httpMux.Handle("/", rqliteProxy) + } else { + httpMux.Handle("/isp-etp/", etpSrv) + } return httpMux } diff --git a/service/rqlite/credentials.go b/service/rqlite/credentials.go new file mode 100644 index 0000000..1b4d434 --- /dev/null +++ b/service/rqlite/credentials.go @@ -0,0 +1,23 @@ +package rqlite + +import ( + "bytes" + "github.com/pkg/errors" + "github.com/rqlite/rqlite/v8/auth" + "github.com/txix-open/isp-kit/json" +) + +func credentialsStore(credentials []auth.Credential) (*auth.CredentialsStore, error) { + data, err := json.Marshal(credentials) + if err != nil { + return nil, errors.WithMessage(err, "marshal credentials") + } + + store := auth.NewCredentialsStore() + err = store.Load(bytes.NewReader(data)) + if err != nil { + return nil, errors.WithMessage(err, "load rqlite credentials") + } + + return store, nil +} diff --git a/service/rqlite/db/db.go b/service/rqlite/db/db.go index 203c8b8..9097d1e 100644 --- a/service/rqlite/db/db.go +++ b/service/rqlite/db/db.go @@ -14,8 +14,7 @@ type Adapter struct { cli *httpcli.Client } -func Open(ctx context.Context, dsn string, client *httpcli.Client) (*Adapter, error) { - client.GlobalRequestConfig().BaseUrl = dsn +func Open(ctx context.Context, client *httpcli.Client) (*Adapter, error) { db := &Adapter{ cli: client, } diff --git a/service/rqlite/main.go b/service/rqlite/main.go index a2898c1..0e7a686 100644 --- a/service/rqlite/main.go +++ b/service/rqlite/main.go @@ -111,11 +111,15 @@ func main(ctx context.Context, r *Rqlite) error { r.store = str + credStr, err := credentialsStore(r.credentials) + if err != nil { + log.Fatalf("failed to create credentials: %s", err.Error()) + } // Get any credential store. - credStr, err := credentialStore(cfg) + /*credStr, err := credentialStore(cfg) if err != nil { log.Fatalf("failed to get credential store: %s", err.Error()) - } + }*/ // Create cluster service now, so nodes will be able to learn information about each other. clstrServ, err := clusterService(cfg, mux.Listen(cluster.MuxClusterHeader), str, str, credStr) diff --git a/service/rqlite/runner.go b/service/rqlite/runner.go index c77187c..8cc89c4 100644 --- a/service/rqlite/runner.go +++ b/service/rqlite/runner.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "fmt" + "github.com/rqlite/rqlite/v8/auth" + "github.com/txix-open/isp-kit/http/httpcli" "time" "github.com/pkg/errors" @@ -17,7 +19,9 @@ var ( ) type Rqlite struct { - cfg *config.Config + cfg *config.Config + internalClientCredential *httpcli.BasicAuth + credentials []auth.Credential localHttpAddr string store *store.Store @@ -25,10 +29,16 @@ type Rqlite struct { cancel context.CancelFunc } -func New(cfg *config.Config) *Rqlite { +func New( + cfg *config.Config, + internalClientCredential *httpcli.BasicAuth, + credentials []auth.Credential, +) *Rqlite { return &Rqlite{ - cfg: cfg, - closed: make(chan struct{}), + cfg: cfg, + internalClientCredential: internalClientCredential, + credentials: credentials, + closed: make(chan struct{}), } } @@ -70,7 +80,20 @@ func (r *Rqlite) SqlDB() (*sql.DB, error) { } func (r *Rqlite) Dsn() string { - return fmt.Sprintf("http://%s", r.localHttpAddr) + return fmt.Sprintf( + "http://%s:%s@%s", + r.internalClientCredential.Username, + r.internalClientCredential.Password, + r.localHttpAddr, + ) +} + +func (r *Rqlite) LocalHttpAddr() string { + return r.localHttpAddr +} + +func (r *Rqlite) InternalClientCredential() *httpcli.BasicAuth { + return r.internalClientCredential } func (r *Rqlite) Close() error { diff --git a/service/startup/service.go b/service/startup/service.go index b2924dd..0646a8f 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -33,6 +33,7 @@ const ( type Service struct { boot *bootstrap.Bootstrap + cfg conf.Local rqlite *rqlite.Rqlite grpcSrv *grpc.Server httpSrv *http.Server @@ -45,34 +46,41 @@ type Service struct { cleanEventWorker *worker.Worker } -func New(boot *bootstrap.Bootstrap) *Service { - rqlite := rqlite.New(boot.App.Config()) +func New(boot *bootstrap.Bootstrap) (*Service, error) { + localConfig := conf.Local{} + err := boot.App.Config().Read(&localConfig) + if err != nil { + return nil, errors.WithMessage(err, "read local config") + } + + logLevelValue := boot.App.Config().Optional().String("logLevel", "info") + var logLevel log.Level + err = logLevel.UnmarshalText([]byte(logLevelValue)) + if err != nil { + return nil, errors.WithMessage(err, "unmarshal log level") + } + boot.App.Logger().SetLevel(logLevel) + + internalClientCredential, err := internalClientCredentials(localConfig) + if err != nil { + return nil, errors.WithMessage(err, "get internal client credentials") + } + + rqlite := rqlite.New(boot.App.Config(), internalClientCredential, localConfig.Credentials) + return &Service{ boot: boot, + cfg: localConfig, rqlite: rqlite, grpcSrv: grpc.DefaultServer(), httpSrv: http.NewServer(boot.App.Logger()), clusterCli: boot.ClusterCli, logger: sentry.WrapErrorLogger(boot.App.Logger(), boot.SentryHub), - } + }, nil } // nolint:funlen func (s *Service) Run(ctx context.Context) error { - logLevelValue := s.boot.App.Config().Optional().String("logLevel", "info") - var logLevel log.Level - err := logLevel.UnmarshalText([]byte(logLevelValue)) - if err != nil { - return errors.WithMessage(err, "parse log level") - } - s.boot.App.Logger().SetLevel(logLevel) - - localConfig := conf.Local{} - err = s.boot.App.Config().Read(&localConfig) - if err != nil { - return errors.WithMessage(err, "read local config") - } - go func() { s.logger.Debug(ctx, "running embedded rqlite...") err := s.rqlite.Run(ctx) @@ -83,7 +91,7 @@ func (s *Service) Run(ctx context.Context) error { time.Sleep(1 * time.Second) // optimistically wait for store initialization s.logger.Debug(ctx, fmt.Sprintf("waiting for cluster startup for %s...", waitForLeaderTimeout)) - err = s.rqlite.WaitForLeader(waitForLeaderTimeout) + err := s.rqlite.WaitForLeader(waitForLeaderTimeout) if err != nil { return errors.WithMessage(err, "wait for leader") } @@ -98,16 +106,21 @@ func (s *Service) Run(ctx context.Context) error { s.logger.Debug(ctx, "is not a leader") } + rqliteClient := httpclix.Default(httpcli.WithMiddlewares(middlewares.SqlOperationMiddleware())) + rqliteClient.GlobalRequestConfig().BaseUrl = fmt.Sprintf("http://%s", s.rqlite.LocalHttpAddr()) + rqliteClient.GlobalRequestConfig().BasicAuth = s.rqlite.InternalClientCredential() db, err := db.Open( ctx, - s.rqlite.Dsn(), - httpclix.Default(httpcli.WithMiddlewares(middlewares.SqlOperationMiddleware())), + rqliteClient, ) if err != nil { return errors.WithMessage(err, "dial to embedded rqlite") } - config := assembly.NewLocator(s.logger, db).Config() + config := assembly.NewLocator(s.logger, db, assembly.LocalConfig{ + Local: s.cfg, + RqliteAddress: s.rqlite.LocalHttpAddr(), + }).Config() s.etpSrv = config.EtpSrv s.handleEventWorker = config.HandleEventWorker s.cleanEventWorker = config.CleanEventWorker @@ -126,7 +139,7 @@ func (s *Service) Run(ctx context.Context) error { }() go func() { - httpPort := localConfig.ConfigServiceAddress.Port + httpPort := s.cfg.ConfigServiceAddress.Port s.logger.Debug(ctx, fmt.Sprintf("starting http server on 0.0.0.0:%s", httpPort)) err := s.httpSrv.ListenAndServe(fmt.Sprintf("0.0.0.0:%s", httpPort)) if err != nil { @@ -193,3 +206,12 @@ func (s *Service) leaderStartup(ctx context.Context) error { return nil } + +func internalClientCredentials(cfg conf.Local) (*httpcli.BasicAuth, error) { + for _, credential := range cfg.Credentials { + if credential.Username == cfg.InternalClientCredential { + return &httpcli.BasicAuth{Username: credential.Username, Password: credential.Password}, nil + } + } + return nil, errors.Errorf("internal client credential '%s' not found", cfg.InternalClientCredential) +} diff --git a/tests/acceptance_test.go b/tests/acceptance_test.go index b7925fb..0e6fd37 100644 --- a/tests/acceptance_test.go +++ b/tests/acceptance_test.go @@ -32,14 +32,15 @@ func TestAcceptance(t *testing.T) { boot.App.Config().Set("rqlite.DataPath", dataPath) logger := boot.App.Logger() - startup := startup.New(boot) + startup, err := startup.New(boot) + require.NoError(err) t.Cleanup(func() { for _, closer := range startup.Closers() { err := closer.Close() require.NoError(err) } }) - err := startup.Run(context.Background()) + err = startup.Run(context.Background()) require.NoError(err) time.Sleep(1 * time.Second) From 53460303234a33c77e411844ad066961d9017c0f Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Mon, 28 Oct 2024 21:44:54 +0300 Subject: [PATCH 30/40] Fix maintaince proxy --- cmd/migrate/go.mod | 33 +++++++++++++++++- cmd/migrate/go.sum | 71 +++++++++++++++++++++++++++++++++++--- service/rqlite/runner.go | 2 +- service/startup/service.go | 16 +++++---- 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index f5f700e..6d5ffbc 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -15,31 +15,62 @@ require ( require ( github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coder/websocket v1.8.12 // indirect + github.com/gabriel-vasile/mimetype v1.4.6 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.7.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.60.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/txix-open/bellows v1.2.0 // indirect github.com/txix-open/etp/v3 v3.2.0 // indirect + github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 6f185d2..811843e 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -2,8 +2,14 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -12,8 +18,19 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= +github.com/getsentry/sentry-go v0.29.1 h1:DyZuChN8Hz3ARxGVV8ePaNXh1dQ7d76AiB117xcREwA= +github.com/getsentry/sentry-go v0.29.1/go.mod h1:x3AtIzN01d6SiWkderzaH28Tm0lgkafpJ5Bm3li39O0= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -22,9 +39,13 @@ github.com/go-resty/resty/v2 v2.15.3 h1:bqff+hcqAflpiF591hhJzNdkRsFhlB96CYfBwSFv github.com/go-resty/resty/v2 v2.15.3/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= @@ -42,10 +63,16 @@ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -67,6 +94,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -76,10 +105,18 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.22.1 h1:2zICEfr1O3yTP9BRZMGPj7qFxQ+ik6yeo+z1LMuioLc= github.com/pressly/goose/v3 v3.22.1/go.mod h1:xtMpbstWyCpyH+0cxLTMCENWBG+0CSxvTsXhW95d5eo= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -109,6 +146,24 @@ github.com/txix-open/isp-kit v1.39.0 h1:NBV2oJJefH/qt2pcPrqEmYccJedjPaVCwr6Z955C github.com/txix-open/isp-kit v1.39.0/go.mod h1:R4vwujGjMXLQir8MDDCfsDQa0CWGuZBSMakt5qcybgM= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -125,6 +180,14 @@ golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38 h1:2oV8dfuIkM1Ti7DwXc0BJfnwr9csz4TDXI9EmiI+Rbw= +google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38/go.mod h1:vuAjtvlwkDKF6L1GQ0SokiRLCGFfeBUXWr/aFFkHACc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/service/rqlite/runner.go b/service/rqlite/runner.go index 8cc89c4..b08ad3e 100644 --- a/service/rqlite/runner.go +++ b/service/rqlite/runner.go @@ -89,7 +89,7 @@ func (r *Rqlite) Dsn() string { } func (r *Rqlite) LocalHttpAddr() string { - return r.localHttpAddr + return fmt.Sprintf("http://%s", r.localHttpAddr) } func (r *Rqlite) InternalClientCredential() *httpcli.BasicAuth { diff --git a/service/startup/service.go b/service/startup/service.go index 0646a8f..d1ee56b 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -107,7 +107,7 @@ func (s *Service) Run(ctx context.Context) error { } rqliteClient := httpclix.Default(httpcli.WithMiddlewares(middlewares.SqlOperationMiddleware())) - rqliteClient.GlobalRequestConfig().BaseUrl = fmt.Sprintf("http://%s", s.rqlite.LocalHttpAddr()) + rqliteClient.GlobalRequestConfig().BaseUrl = s.rqlite.LocalHttpAddr() rqliteClient.GlobalRequestConfig().BasicAuth = s.rqlite.InternalClientCredential() db, err := db.Open( ctx, @@ -148,12 +148,14 @@ func (s *Service) Run(ctx context.Context) error { }() time.Sleep(1 * time.Second) // wait for http start - go func() { - err = s.clusterCli.Run(ctx, cluster.NewEventHandler()) - if err != nil { - s.boot.Fatal(errors.WithMessage(err, "connect to it's self")) - } - }() + if !s.cfg.MaintenanceMode { + go func() { + err = s.clusterCli.Run(ctx, cluster.NewEventHandler()) + if err != nil { + s.boot.Fatal(errors.WithMessage(err, "connect to it's self")) + } + }() + } return nil } From d4badbb8dbaec5a65cba9e79abc5f9096653235f Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Mon, 28 Oct 2024 22:17:10 +0300 Subject: [PATCH 31/40] Return prev module id --- repository/module.go | 13 +++++++------ service/module/service.go | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/repository/module.go b/repository/module.go index 0bc84f3..6252660 100644 --- a/repository/module.go +++ b/repository/module.go @@ -21,23 +21,24 @@ func NewModule(db db.DB) Module { } } -func (r Module) Upsert(ctx context.Context, module entity.Module) error { +func (r Module) Upsert(ctx context.Context, module entity.Module) (string, error) { ctx = sql_metrics.OperationLabelToContext(ctx, "Module.Upsert") query, args, err := squirrel.Insert(Table("module")). Columns("id", "name", "last_connected_at"). Values(module.Id, module.Name, squirrel.Expr("unixepoch()")). - Suffix(`on conflict (name) do update set last_connected_at = unixepoch()`). + Suffix(`on conflict (name) do update set last_connected_at = unixepoch() returning id`). ToSql() if err != nil { - return errors.WithMessage(err, "build query") + return "", errors.WithMessage(err, "build query") } - _, err = r.db.Exec(ctx, query, args...) + result := make(map[string]string) + err = r.db.SelectRow(ctx, &result, query, args...) if err != nil { - return errors.WithMessagef(err, "select: %s", query) + return "", errors.WithMessagef(err, "select: %s", query) } - return nil + return result["id"], nil } func (r Module) SetDisconnectedAtNow( diff --git a/service/module/service.go b/service/module/service.go index 76527dc..0e3ed2e 100644 --- a/service/module/service.go +++ b/service/module/service.go @@ -23,7 +23,7 @@ const ( ) type Repo interface { - Upsert(ctx context.Context, module entity.Module) error + Upsert(ctx context.Context, module entity.Module) (string, error) SetDisconnectedAtNow( ctx context.Context, moduleId string, @@ -95,7 +95,7 @@ func (s Service) OnConnect(ctx context.Context, conn *etp.Conn, moduleName strin Id: moduleId, Name: moduleName, } - err := s.moduleRepo.Upsert(ctx, module) + moduleId, err := s.moduleRepo.Upsert(ctx, module) if err != nil { return errors.WithMessage(err, "upsert module in store") } From 43a1a206d5d1d401b6d59901b3e6cf552abd9882 Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Tue, 29 Oct 2024 12:04:52 +0300 Subject: [PATCH 32/40] Bump dependencies --- cmd/migrate/go.mod | 2 +- cmd/migrate/go.sum | 4 ++-- cmd/migrate/main.go | 2 +- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index 6d5ffbc..c1bc480 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -46,7 +46,7 @@ require ( github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect - github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/txix-open/bellows v1.2.0 // indirect diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 811843e..b7166ea 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -131,8 +131,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index e9dba18..8c5fa48 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -87,7 +87,7 @@ func main() { Name: module.Name, CreatedAt: xtypes.Time(module.CreatedAt), } - err := moduleRepo.Upsert(ctx, result) + _, err := moduleRepo.Upsert(ctx, result) if err != nil { panic(errors.WithMessagef(err, "err upserting module: %s", module.Name)) } diff --git a/go.mod b/go.mod index 02d0868..b8a38e6 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,10 @@ require ( github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.22.1 github.com/prometheus/client_golang v1.20.5 - github.com/rqlite/gorqlite v0.0.0-20240808172217-12ae7d03ef19 - github.com/rqlite/rqlite/v8 v8.30.0 + github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d + github.com/rqlite/rqlite/v8 v8.32.4 github.com/stretchr/testify v1.9.0 - github.com/tidwall/gjson v1.17.1 + github.com/tidwall/gjson v1.18.0 github.com/txix-open/etp/v3 v3.2.0 github.com/txix-open/isp-kit v1.39.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -60,7 +60,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/rqlite/go-sqlite3 v1.35.0 // indirect + github.com/rqlite/go-sqlite3 v1.36.0 // indirect github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 // indirect github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd // indirect github.com/sethvargo/go-retry v0.3.0 // indirect @@ -72,7 +72,7 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/bbolt v1.3.11 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 // indirect diff --git a/go.sum b/go.sum index 9660a44..6e3c4f6 100644 --- a/go.sum +++ b/go.sum @@ -195,14 +195,14 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/rqlite/go-sqlite3 v1.35.0 h1:NTG3mmwZly6Bhfe8oi7HNtI9jUDIs6XtXGZZoVa/eRA= -github.com/rqlite/go-sqlite3 v1.35.0/go.mod h1:R9H7CatgYBt3c+fSV/5yo2vLh4ZjCB0aMHdkv69fP4A= -github.com/rqlite/gorqlite v0.0.0-20240808172217-12ae7d03ef19 h1:uuWunw893WVwpSg4kNBuS6swgABwc+rwInVtwR5E3eM= -github.com/rqlite/gorqlite v0.0.0-20240808172217-12ae7d03ef19/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= +github.com/rqlite/go-sqlite3 v1.36.0 h1:dNW9Hn4v3HVVlO+q6QmQC5glZYIWFOzOWUjrKOxK3cE= +github.com/rqlite/go-sqlite3 v1.36.0/go.mod h1:R9H7CatgYBt3c+fSV/5yo2vLh4ZjCB0aMHdkv69fP4A= +github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d h1:c88ius/WcN19inn14R+X2EQCFjjAu92txgdxNNnGxDI= +github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 h1:NZ62M+kT0JqhyFUMc8I4SMmfmD4NGJxhb2ePJQXjryc= github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48/go.mod h1:CRnsxgy5G8fAf5J+AM0yrsSdxXHKkIYOaq2sm+Q4DYc= -github.com/rqlite/rqlite/v8 v8.30.0 h1:93htyRh3NQzHEavk4/r2AJyA5w2GturOOn7X4CAfVeU= -github.com/rqlite/rqlite/v8 v8.30.0/go.mod h1:j7PkXMRmrCZh24FUBbHL5788UKxvgDG6s0na8WGXIvM= +github.com/rqlite/rqlite/v8 v8.32.4 h1:z3C4kDLGHVQ9qz7vbgDkrrEyXfJtAX+JoTy0u0+71DA= +github.com/rqlite/rqlite/v8 v8.32.4/go.mod h1:cKc8BMsAiHwj7h+pkVf1WdND9viCFbktFudDIA6hWbs= github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd h1:wW6BtayFoKaaDeIvXRE3SZVPOscSKlYD+X3bB749+zk= github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= @@ -223,8 +223,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -254,8 +254,8 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= -go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= From 3f60267bdc5e49fa59646b20e9464a2f5b0eb5ab Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Tue, 29 Oct 2024 12:35:31 +0300 Subject: [PATCH 33/40] Fix migration --- cmd/migrate/main.go | 15 +++++++++------ service/rqlite/goose_store/dialect.go | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index 8c5fa48..1454ad0 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -2,6 +2,8 @@ package main import ( "fmt" + "github.com/pressly/goose/v3" + "github.com/pressly/goose/v3/database" "github.com/txix-open/isp-kit/config" "github.com/txix-open/isp-kit/dbx" "github.com/txix-open/isp-kit/http/httpcli" @@ -64,8 +66,12 @@ func main() { } defer dbClient.Close() - mr := migration.NewRunner(migration.DialectSqlite3, migrations.Migrations, logger) - err = mr.Run(ctx, dbClient.DB.DB) + migrationStore, err := database.NewStore(migration.DialectSqlite3, repository.Table("goose_db_version")) + if err != nil { + panic(errors.WithMessage(err, "create store")) + } + mr := migration.NewRunner("", migrations.Migrations, logger) + err = mr.Run(ctx, dbClient.DB.DB, goose.WithStore(migrationStore)) if err != nil { panic(errors.WithMessage(err, "err connecting to db")) } @@ -87,10 +93,7 @@ func main() { Name: module.Name, CreatedAt: xtypes.Time(module.CreatedAt), } - _, err := moduleRepo.Upsert(ctx, result) - if err != nil { - panic(errors.WithMessagef(err, "err upserting module: %s", module.Name)) - } + _, _ = moduleRepo.Upsert(ctx, result) } logger.Info(ctx, "modules migrated") diff --git a/service/rqlite/goose_store/dialect.go b/service/rqlite/goose_store/dialect.go index 1735169..4f9e82d 100644 --- a/service/rqlite/goose_store/dialect.go +++ b/service/rqlite/goose_store/dialect.go @@ -12,7 +12,7 @@ func (s *Rqlite) CreateTable(tableName string) string { version_id INTEGER NOT NULL, is_applied INTEGER NOT NULL, tstamp DATETIME DEFAULT (datetime('now')) - )` + ) if not exists` return fmt.Sprintf(q, tableName) } From cb8c66849ae6d2028680177daf7dbb32f2e43881 Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Tue, 29 Oct 2024 12:48:33 +0300 Subject: [PATCH 34/40] Fix migration --- service/rqlite/goose_store/dialect.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/rqlite/goose_store/dialect.go b/service/rqlite/goose_store/dialect.go index 4f9e82d..c22952f 100644 --- a/service/rqlite/goose_store/dialect.go +++ b/service/rqlite/goose_store/dialect.go @@ -7,12 +7,12 @@ import ( type Rqlite struct{} func (s *Rqlite) CreateTable(tableName string) string { - q := `CREATE TABLE %s ( + q := `CREATE TABLE if not exists %s ( id INTEGER PRIMARY KEY AUTOINCREMENT, version_id INTEGER NOT NULL, is_applied INTEGER NOT NULL, tstamp DATETIME DEFAULT (datetime('now')) - ) if not exists` + )` return fmt.Sprintf(q, tableName) } From 68c8681a9bfce4db57e28a75c04abc11570690b0 Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Tue, 29 Oct 2024 17:41:03 +0300 Subject: [PATCH 35/40] Keep only last N config versions --- assembly/locator.go | 8 +++--- changelog.md | 2 ++ cmd/migrate/main.go | 7 ++---- conf/config.yml | 3 ++- conf/local.go | 1 + repository/config_history.go | 19 ++++++++++++++ service/api/config.go | 37 +++++++++++++-------------- service/api/config_history.go | 47 ++++++++++++++++++++++++++++++++--- 8 files changed, 91 insertions(+), 33 deletions(-) create mode 100644 changelog.md diff --git a/assembly/locator.go b/assembly/locator.go index 3eb5a34..6167cec 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -94,6 +94,9 @@ func (l Locator) Config() Config { ) moduleController := controller.NewModule(moduleService, l.logger) + configHistoryApiService := apisvs.NewConfigHistory(configHistoryRepo, l.cfg.Local.KeepConfigVersions, l.logger) + configHistoryController := api.NewConfigHistory(configHistoryApiService) + moduleApiService := apisvs.NewModule(moduleRepo, backendRepo, configSchemaRepo) moduleApiController := api.NewModule(moduleApiService) @@ -102,13 +105,10 @@ func (l Locator) Config() Config { moduleRepo, configSchemaRepo, eventRepo, - configHistoryRepo, + configHistoryApiService, ) configApiController := api.NewConfig(configApiService) - configHistoryApiService := apisvs.NewConfigHistory(configHistoryRepo) - configHistoryController := api.NewConfigHistory(configHistoryApiService) - configSchemaApiService := apisvs.NewConfigSchema(configSchemaRepo) configSchemaController := api.NewConfigSchema(configSchemaApiService) diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..224cc9b --- /dev/null +++ b/changelog.md @@ -0,0 +1,2 @@ +## v3.0.2 +* Новая реализация модуля на базе rqlite \ No newline at end of file diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index 1454ad0..bce9b58 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -3,13 +3,13 @@ package main import ( "fmt" "github.com/pressly/goose/v3" - "github.com/pressly/goose/v3/database" "github.com/txix-open/isp-kit/config" "github.com/txix-open/isp-kit/dbx" "github.com/txix-open/isp-kit/http/httpcli" "github.com/txix-open/isp-kit/http/httpclix" "github.com/txix-open/isp-kit/log" "github.com/txix-open/isp-kit/validator" + "isp-config-service/service/rqlite/goose_store" "os" "time" @@ -66,10 +66,7 @@ func main() { } defer dbClient.Close() - migrationStore, err := database.NewStore(migration.DialectSqlite3, repository.Table("goose_db_version")) - if err != nil { - panic(errors.WithMessage(err, "create store")) - } + migrationStore := goose_store.NewStore(dbClient.DB.DB) mr := migration.NewRunner("", migrations.Migrations, logger) err = mr.Run(ctx, dbClient.DB.DB, goose.WithStore(migrationStore)) if err != nil { diff --git a/conf/config.yml b/conf/config.yml index 1472ba5..6e22c42 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -15,11 +15,12 @@ logLevel: debug maintenanceMode: false +keepConfigVersions: 64 + credentials: - username: default password: 38kPk0upqI5j perms: ["all"] - internalClientCredential: default rqlite: diff --git a/conf/local.go b/conf/local.go index 3c13836..706a181 100644 --- a/conf/local.go +++ b/conf/local.go @@ -9,5 +9,6 @@ type Local struct { ConfigServiceAddress bootstrap.ConfigServiceAddr MaintenanceMode bool Credentials []auth.Credential + KeepConfigVersions int `validate:"required"` InternalClientCredential string `validate:"required"` } diff --git a/repository/config_history.go b/repository/config_history.go index 0974d20..606cc3f 100644 --- a/repository/config_history.go +++ b/repository/config_history.go @@ -62,3 +62,22 @@ func (r ConfigHistory) Insert(ctx context.Context, history entity.ConfigHistory) return nil } + +func (r ConfigHistory) DeleteOld(ctx context.Context, configId string, keepVersions int) (int, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "ConfigHistory.DeleteOld") + + query := fmt.Sprintf(`delete from %s where config_id = ? and id not in ( + select id from %s where config_id = ? order by version desc limit ? + )`, Table("config_history"), Table("config_history")) + result, err := r.db.Exec(ctx, query, configId, configId, keepVersions) + if err != nil { + return 0, errors.WithMessagef(err, "exec: %s", query) + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return 0, errors.WithMessage(err, "get rows affected") + } + + return int(rowsAffected), nil +} diff --git a/service/api/config.go b/service/api/config.go index 13c646c..1ac29c5 100644 --- a/service/api/config.go +++ b/service/api/config.go @@ -26,12 +26,16 @@ type EventRepo interface { Insert(ctx context.Context, event entity.Event) error } +type ConfigHistoryService interface { + OnUpdateConfig(ctx context.Context, oldConfig entity.Config) error +} + type Config struct { - configRepo ConfigRepo - moduleRepo ModuleRepo - schemaRepo SchemaRepo - eventRepo EventRepo - configHistoryRepo ConfigHistoryRepo + configRepo ConfigRepo + moduleRepo ModuleRepo + schemaRepo SchemaRepo + eventRepo EventRepo + configHistoryService ConfigHistoryService } func NewConfig( @@ -39,14 +43,14 @@ func NewConfig( moduleRepo ModuleRepo, schemaRepo SchemaRepo, eventRepo EventRepo, - configHistoryRepo ConfigHistoryRepo, + configHistoryService ConfigHistoryService, ) Config { return Config{ - configRepo: configRepo, - moduleRepo: moduleRepo, - schemaRepo: schemaRepo, - eventRepo: eventRepo, - configHistoryRepo: configHistoryRepo, + configRepo: configRepo, + moduleRepo: moduleRepo, + schemaRepo: schemaRepo, + eventRepo: eventRepo, + configHistoryService: configHistoryService, } } @@ -137,16 +141,9 @@ func (c Config) CreateUpdateConfig( return nil, entity.ErrConfigConflictUpdate } - history := entity.ConfigHistory{ - Id: uuid.NewString(), - ConfigId: req.Id, - Data: oldConfig.Data, - Version: oldConfig.Version, - AdminId: oldConfig.AdminId, - } - err = c.configHistoryRepo.Insert(ctx, history) + err = c.configHistoryService.OnUpdateConfig(ctx, *oldConfig) if err != nil { - return nil, errors.WithMessage(err, "save config history") + return nil, errors.WithMessage(err, "change config history") } if oldConfig.Active { diff --git a/service/api/config_history.go b/service/api/config_history.go index 701084c..76681f5 100644 --- a/service/api/config_history.go +++ b/service/api/config_history.go @@ -2,6 +2,9 @@ package api import ( "context" + "fmt" + "github.com/google/uuid" + "github.com/txix-open/isp-kit/log" "time" "github.com/pkg/errors" @@ -13,15 +16,24 @@ type ConfigHistoryRepo interface { Delete(ctx context.Context, id string) error GetByConfigId(ctx context.Context, configId string) ([]entity.ConfigHistory, error) Insert(ctx context.Context, history entity.ConfigHistory) error + DeleteOld(ctx context.Context, configId string, keepVersions int) (int, error) } type ConfigHistory struct { - repo ConfigHistoryRepo + repo ConfigHistoryRepo + keepVersions int + logger log.Logger } -func NewConfigHistory(repo ConfigHistoryRepo) ConfigHistory { +func NewConfigHistory( + repo ConfigHistoryRepo, + keepVersions int, + logger log.Logger, +) ConfigHistory { return ConfigHistory{ - repo: repo, + repo: repo, + keepVersions: keepVersions, + logger: logger, } } @@ -52,3 +64,32 @@ func (s ConfigHistory) Delete(ctx context.Context, id string) error { } return nil } + +func (s ConfigHistory) OnUpdateConfig(ctx context.Context, oldConfig entity.Config) error { + history := entity.ConfigHistory{ + Id: uuid.NewString(), + ConfigId: oldConfig.Id, + Data: oldConfig.Data, + Version: oldConfig.Version, + AdminId: oldConfig.AdminId, + } + err := s.repo.Insert(ctx, history) + if err != nil { + return errors.WithMessage(err, "save config history") + } + + if s.keepVersions < 0 { + return nil + } + + go func() { + deletedCount, err := s.repo.DeleteOld(context.Background(), oldConfig.Id, s.keepVersions) + if err != nil { + s.logger.Error(ctx, errors.WithMessage(err, "delete old config versions")) + } else { + s.logger.Debug(ctx, fmt.Sprintf("delete '%d' old config versions", deletedCount)) + } + }() + + return nil +} From 4f4fb5e6c7db8e64ed55da532ded673c95c7d169 Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Tue, 29 Oct 2024 18:11:36 +0300 Subject: [PATCH 36/40] Clean events only on leader --- assembly/locator.go | 27 +++++++++++++++++++-------- service/event/cleaner.go | 31 ++++++++++++++++++++++++------- service/startup/service.go | 5 +++-- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/assembly/locator.go b/assembly/locator.go index 6167cec..70c3044 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -36,16 +36,27 @@ type LocalConfig struct { } type Locator struct { - db db.DB - cfg LocalConfig - logger log.Logger + db db.DB + cfg LocalConfig + leaderChecker LeaderChecker + logger log.Logger } -func NewLocator(logger log.Logger, db db.DB, cfg LocalConfig) Locator { +type LeaderChecker interface { + IsLeader() bool +} + +func NewLocator( + db db.DB, + leaderChecker LeaderChecker, + cfg LocalConfig, + logger log.Logger, +) Locator { return Locator{ - db: db, - cfg: cfg, - logger: logger, + db: db, + leaderChecker: leaderChecker, + cfg: cfg, + logger: logger, } } @@ -135,7 +146,7 @@ func (l Locator) Config() Config { handleEventJob := event.NewWorker(eventRepo, eventHandler, l.logger) handleEventWorker := worker.New(handleEventJob, worker.WithInterval(handleEventsInterval)) - cleanerJob := event.NewCleaner(eventRepo, eventTtl, l.logger) + cleanerJob := event.NewCleaner(eventRepo, l.leaderChecker, eventTtl, l.logger) cleanEventWorker := worker.New(cleanerJob, worker.WithInterval(cleanEventInterval)) return Config{ diff --git a/service/event/cleaner.go b/service/event/cleaner.go index 4a1c20d..fec3088 100644 --- a/service/event/cleaner.go +++ b/service/event/cleaner.go @@ -10,21 +10,38 @@ import ( "isp-config-service/entity/xtypes" ) +type LeaderChecker interface { + IsLeader() bool +} + type Cleaner struct { - repo Repo - eventTtl time.Duration - logger log.Logger + repo Repo + leaderChecker LeaderChecker + eventTtl time.Duration + logger log.Logger } -func NewCleaner(repo Repo, eventTtl time.Duration, logger log.Logger) Cleaner { +func NewCleaner( + repo Repo, + leaderChecker LeaderChecker, + eventTtl time.Duration, + logger log.Logger, +) Cleaner { return Cleaner{ - repo: repo, - eventTtl: eventTtl, - logger: logger, + repo: repo, + leaderChecker: leaderChecker, + eventTtl: eventTtl, + logger: logger, } } func (c Cleaner) Do(ctx context.Context) { + ctx = log.ToContext(ctx, log.String("worker", "eventCleaner")) + if !c.leaderChecker.IsLeader() { + c.logger.Debug(ctx, "is not a leader, skip work") + return + } + deleteBefore := time.Now().Add(-c.eventTtl) deleted, err := c.repo.DeleteByCreatedAt(ctx, xtypes.Time(deleteBefore)) if err != nil { diff --git a/service/startup/service.go b/service/startup/service.go index d1ee56b..1a97a18 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -117,10 +117,11 @@ func (s *Service) Run(ctx context.Context) error { return errors.WithMessage(err, "dial to embedded rqlite") } - config := assembly.NewLocator(s.logger, db, assembly.LocalConfig{ + cfg := assembly.LocalConfig{ Local: s.cfg, RqliteAddress: s.rqlite.LocalHttpAddr(), - }).Config() + } + config := assembly.NewLocator(db, s.rqlite, cfg, s.logger).Config() s.etpSrv = config.EtpSrv s.handleEventWorker = config.HandleEventWorker s.cleanEventWorker = config.CleanEventWorker From 1342f0dcf2551f93df9a5f71e32c7155192e456c Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Tue, 29 Oct 2024 21:54:01 +0300 Subject: [PATCH 37/40] Close cluster client --- cmd/migrate/main.go | 1 + service/startup/service.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index bce9b58..7bf02a3 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -183,6 +183,7 @@ func main() { Header("Content-Type", "application/octet-stream"). RequestBody(sqliteFile). StatusCodeToError(). + Timeout(60 * time.Second). DoWithoutResponse(ctx) if err != nil { panic(errors.WithMessage(err, "load sqlite file")) diff --git a/service/startup/service.go b/service/startup/service.go index 1a97a18..b071f46 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -163,6 +163,7 @@ func (s *Service) Run(ctx context.Context) error { func (s *Service) Closers() []app.Closer { return []app.Closer{ + s.clusterCli, app.CloserFunc(func() error { s.grpcSrv.Shutdown() return nil From 7aa2805f4c103157f71cdcb4b5eed215d46c61a0 Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Fri, 1 Nov 2024 18:41:17 +0300 Subject: [PATCH 38/40] Clean phantom backends --- .version | 2 +- assembly/locator.go | 58 ++++--- changelog.md | 4 + cmd/migrate/go.mod | 4 +- cmd/migrate/go.sum | 8 +- conf/local.go | 5 + controller/module.go | 4 +- entity/backend.go | 19 ++- go.mod | 10 +- go.sum | 20 +-- helpers/helpers.go | 8 +- middlewares/ws_log.go | 4 +- .../20241101112923_recreate_backends.sql | 23 +++ repository/backend.go | 71 ++++++-- routes/routes.go | 2 +- service/api/module.go | 2 +- service/module/backend/backend.go | 156 ++++++++++++++++++ service/module/backend/cleaner.go | 39 +++++ service/module/emitter.go | 4 +- service/module/service.go | 64 ++----- service/startup/service.go | 43 ++--- service/subscription/service.go | 2 +- 22 files changed, 397 insertions(+), 155 deletions(-) create mode 100644 migrations/20241101112923_recreate_backends.sql create mode 100644 service/module/backend/backend.go create mode 100644 service/module/backend/cleaner.go diff --git a/.version b/.version index b502146..fd2a018 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -3.0.2 +3.1.0 diff --git a/assembly/locator.go b/assembly/locator.go index 70c3044..59ff6df 100644 --- a/assembly/locator.go +++ b/assembly/locator.go @@ -1,13 +1,15 @@ package assembly import ( + "context" "isp-config-service/conf" + "isp-config-service/service/module/backend" "net/http" "net/http/httputil" "net/url" "time" - "github.com/txix-open/etp/v3" + "github.com/txix-open/etp/v4" "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/grpc/endpoint" "github.com/txix-open/isp-kit/log" @@ -24,10 +26,11 @@ import ( ) const ( - wsReadLimit = 4 * 1024 * 1024 - handleEventsInterval = 500 * time.Millisecond - cleanEventInterval = 60 * time.Second - eventTtl = 60 * time.Second + wsReadLimit = 4 * 1024 * 1024 + handleEventsInterval = 500 * time.Millisecond + cleanEventInterval = 60 * time.Second + cleanPhantomBackendsInterval = 5 * time.Minute + eventTtl = 60 * time.Second ) type LocalConfig struct { @@ -46,6 +49,10 @@ type LeaderChecker interface { IsLeader() bool } +type OwnBackendsCleaner interface { + DeleteOwnBackends(ctx context.Context) +} + func NewLocator( db db.DB, leaderChecker LeaderChecker, @@ -61,15 +68,17 @@ func NewLocator( } type Config struct { - GrpcMux *grpc.Mux - HttpMux http.Handler - EtpSrv *etp.Server - HandleEventWorker *worker.Worker - CleanEventWorker *worker.Worker + GrpcMux *grpc.Mux + HttpMux http.Handler + EtpSrv *etp.Server + HandleEventWorker *worker.Worker + CleanEventWorker *worker.Worker + CleanPhantomBackendWorker *worker.Worker + OwnBackendsCleaner OwnBackendsCleaner } //nolint:funlen -func (l Locator) Config() Config { +func (l Locator) Config() *Config { moduleRepo := repository.NewModule(l.db) backendRepo := repository.NewBackend(l.db) eventRepo := repository.NewEvent(l.db) @@ -93,10 +102,19 @@ func (l Locator) Config() Config { l.logger, ) - moduleService := module.NewService( - moduleRepo, + backendService := backend.NewBackend( backendRepo, eventRepo, + l.cfg.Local.Rqlite.NodeId, + etpSrv.Rooms(), + l.logger, + ) + cleanPhantomBackendJob := backend.NewCleaner(backendService, l.logger) + cleanPhantomBackendWorker := worker.New(cleanPhantomBackendJob, worker.WithInterval(cleanPhantomBackendsInterval)) + + moduleService := module.NewService( + moduleRepo, + backendService, configRepo, configSchemaRepo, subscriptionService, @@ -149,11 +167,13 @@ func (l Locator) Config() Config { cleanerJob := event.NewCleaner(eventRepo, l.leaderChecker, eventTtl, l.logger) cleanEventWorker := worker.New(cleanerJob, worker.WithInterval(cleanEventInterval)) - return Config{ - GrpcMux: grpcMux, - EtpSrv: etpSrv, - HttpMux: httpMux, - HandleEventWorker: handleEventWorker, - CleanEventWorker: cleanEventWorker, + return &Config{ + GrpcMux: grpcMux, + EtpSrv: etpSrv, + HttpMux: httpMux, + HandleEventWorker: handleEventWorker, + CleanEventWorker: cleanEventWorker, + CleanPhantomBackendWorker: cleanPhantomBackendWorker, + OwnBackendsCleaner: backendService, } } diff --git a/changelog.md b/changelog.md index 224cc9b..96c82b7 100644 --- a/changelog.md +++ b/changelog.md @@ -1,2 +1,6 @@ +## v3.1.0 +* добавлено возможность анонсировать несколько бекендов с одинаковыми адресами +* реализован механизм поиска и очистки фантомных бекендов (те который отключились, но остались висеть в таблице маршрутизации) +* добавлена очистка таблицы маршрутизации в момент отключения самого модуля ## v3.0.2 * Новая реализация модуля на базе rqlite \ No newline at end of file diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index c1bc480..1fa14ea 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -9,7 +9,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.22 github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.22.1 - github.com/txix-open/isp-kit v1.39.0 + github.com/txix-open/isp-kit v1.39.1 isp-config-service v0.0.0-00010101000000-000000000000 ) @@ -50,7 +50,7 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/txix-open/bellows v1.2.0 // indirect - github.com/txix-open/etp/v3 v3.2.0 // indirect + github.com/txix-open/etp/v4 v4.0.1 // indirect github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 // indirect go.opentelemetry.io/otel v1.31.0 // indirect diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index b7166ea..79eb56c 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -140,10 +140,10 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgjeY= github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o= -github.com/txix-open/etp/v3 v3.2.0 h1:EgTchT8VtCYV1iEo/y4jII3/NybrXiQDE0y6pmS350M= -github.com/txix-open/etp/v3 v3.2.0/go.mod h1:lgen0TJmiturgBhvlxH1/PQ7iLL/OebKbxPAQYkuKI0= -github.com/txix-open/isp-kit v1.39.0 h1:NBV2oJJefH/qt2pcPrqEmYccJedjPaVCwr6Z955C+pg= -github.com/txix-open/isp-kit v1.39.0/go.mod h1:R4vwujGjMXLQir8MDDCfsDQa0CWGuZBSMakt5qcybgM= +github.com/txix-open/etp/v4 v4.0.1 h1:5VSYgGsvnMz0tvUHsjVWFETI0qWBTCF1O3ARCHaCIgA= +github.com/txix-open/etp/v4 v4.0.1/go.mod h1:FIu7TUDdwcgmtmF6tFTV1bGA2fRY/iVaPQdsGzvqYc8= +github.com/txix-open/isp-kit v1.39.1 h1:10zEILNQSPY7naiuaQkIKNS2/fnWxuE9bz+dTiAptXw= +github.com/txix-open/isp-kit v1.39.1/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= diff --git a/conf/local.go b/conf/local.go index 706a181..93d534b 100644 --- a/conf/local.go +++ b/conf/local.go @@ -11,4 +11,9 @@ type Local struct { Credentials []auth.Credential KeepConfigVersions int `validate:"required"` InternalClientCredential string `validate:"required"` + Rqlite Rqlite +} + +type Rqlite struct { + NodeId string `validate:"required"` } diff --git a/controller/module.go b/controller/module.go index 815f6cf..95188c4 100644 --- a/controller/module.go +++ b/controller/module.go @@ -2,10 +2,10 @@ package controller import ( "context" + "github.com/txix-open/etp/v4" "github.com/pkg/errors" - "github.com/txix-open/etp/v3" - "github.com/txix-open/etp/v3/msg" + "github.com/txix-open/etp/v4/msg" "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/json" "github.com/txix-open/isp-kit/log" diff --git a/entity/backend.go b/entity/backend.go index 845042a..3e39a26 100644 --- a/entity/backend.go +++ b/entity/backend.go @@ -6,13 +6,14 @@ import ( ) type Backend struct { - ModuleId string `json:"module_id"` - Address string `json:"address"` - Version string `json:"version"` - LibVersion string `json:"lib_version"` - ModuleName string `json:"module_name"` - Endpoints xtypes.Json[[]cluster.EndpointDescriptor] `json:"endpoints"` - RequiredModules xtypes.Json[[]cluster.ModuleDependency] `json:"required_modules"` - CreatedAt xtypes.Time `json:"created_at"` - UpdatedAt xtypes.Time `json:"updated_at"` + WsConnectionId string `json:"ws_connection_id"` + ModuleId string `json:"module_id"` + Address string `json:"address"` + Version string `json:"version"` + LibVersion string `json:"lib_version"` + ModuleName string `json:"module_name"` + ConfigServiceNodeId string `json:"config_service_node_id"` + Endpoints xtypes.Json[[]cluster.EndpointDescriptor] `json:"endpoints"` + RequiredModules xtypes.Json[[]cluster.ModuleDependency] `json:"required_modules"` + CreatedAt xtypes.Time `json:"created_at"` } diff --git a/go.mod b/go.mod index b8a38e6..1d1f73f 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,11 @@ require ( github.com/pressly/goose/v3 v3.22.1 github.com/prometheus/client_golang v1.20.5 github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d - github.com/rqlite/rqlite/v8 v8.32.4 + github.com/rqlite/rqlite/v8 v8.32.5 github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.18.0 - github.com/txix-open/etp/v3 v3.2.0 - github.com/txix-open/isp-kit v1.39.0 + github.com/txix-open/etp/v4 v4.0.1 + github.com/txix-open/isp-kit v1.39.1 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.8.0 google.golang.org/grpc v1.67.1 @@ -28,7 +28,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coder/websocket v1.8.12 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/fatih/color v1.17.0 // indirect + github.com/fatih/color v1.18.0 // indirect github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/getsentry/sentry-go v0.29.1 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -62,7 +62,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rqlite/go-sqlite3 v1.36.0 // indirect github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 // indirect - github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd // indirect + github.com/rqlite/sql v0.0.0-20241029220113-152a320b02f7 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect diff --git a/go.sum b/go.sum index 6e3c4f6..df76a35 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= @@ -201,10 +201,10 @@ github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d h1:c88ius/WcN19inn github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 h1:NZ62M+kT0JqhyFUMc8I4SMmfmD4NGJxhb2ePJQXjryc= github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48/go.mod h1:CRnsxgy5G8fAf5J+AM0yrsSdxXHKkIYOaq2sm+Q4DYc= -github.com/rqlite/rqlite/v8 v8.32.4 h1:z3C4kDLGHVQ9qz7vbgDkrrEyXfJtAX+JoTy0u0+71DA= -github.com/rqlite/rqlite/v8 v8.32.4/go.mod h1:cKc8BMsAiHwj7h+pkVf1WdND9viCFbktFudDIA6hWbs= -github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd h1:wW6BtayFoKaaDeIvXRE3SZVPOscSKlYD+X3bB749+zk= -github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg= +github.com/rqlite/rqlite/v8 v8.32.5 h1:wEzDX2HrGWMvFkg2fcCmi+rb1wQ1m5yEefUQfOMvMSY= +github.com/rqlite/rqlite/v8 v8.32.5/go.mod h1:a1irqNWEZ+pWuWpgekbScCXp85QtNTJLarN3bVPjn9M= +github.com/rqlite/sql v0.0.0-20241029220113-152a320b02f7 h1:Mnz6yd4FWtiD6bbH9WHFFHfrOM2OYTUTmwrsckRc4W8= +github.com/rqlite/sql v0.0.0-20241029220113-152a320b02f7/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -233,12 +233,12 @@ github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgjeY= github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o= -github.com/txix-open/etp/v3 v3.2.0 h1:EgTchT8VtCYV1iEo/y4jII3/NybrXiQDE0y6pmS350M= -github.com/txix-open/etp/v3 v3.2.0/go.mod h1:lgen0TJmiturgBhvlxH1/PQ7iLL/OebKbxPAQYkuKI0= +github.com/txix-open/etp/v4 v4.0.1 h1:5VSYgGsvnMz0tvUHsjVWFETI0qWBTCF1O3ARCHaCIgA= +github.com/txix-open/etp/v4 v4.0.1/go.mod h1:FIu7TUDdwcgmtmF6tFTV1bGA2fRY/iVaPQdsGzvqYc8= github.com/txix-open/grmq v1.6.0 h1:V4QRb0sI2CMfdMaMsFGvGvUIDGad5vr3khNU7A3LlP8= github.com/txix-open/grmq v1.6.0/go.mod h1:s+NNNv42+32yLpjvl2tpqtyBbMB9e7kKFee3lfa/dyo= -github.com/txix-open/isp-kit v1.39.0 h1:NBV2oJJefH/qt2pcPrqEmYccJedjPaVCwr6Z955C+pg= -github.com/txix-open/isp-kit v1.39.0/go.mod h1:R4vwujGjMXLQir8MDDCfsDQa0CWGuZBSMakt5qcybgM= +github.com/txix-open/isp-kit v1.39.1 h1:10zEILNQSPY7naiuaQkIKNS2/fnWxuE9bz+dTiAptXw= +github.com/txix-open/isp-kit v1.39.1/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I= github.com/txix-open/jsonschema v1.2.0 h1:B8TrdSsPhDvYv67oi/LVqf7/+PuqFlMsNwUzGeHfl1s= github.com/txix-open/jsonschema v1.2.0/go.mod h1:l8YDZ1nvJrw6uxWowSVOxCV/ebiMJyapffW87ZEqH00= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= diff --git a/helpers/helpers.go b/helpers/helpers.go index ec30041..555bbf3 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -1,14 +1,12 @@ package helpers import ( - "net" - "strconv" - "github.com/pkg/errors" - "github.com/txix-open/etp/v3" + "github.com/txix-open/etp/v4" "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/log" "isp-config-service/entity" + "net" ) func ModuleName(conn *etp.Conn) string { @@ -29,6 +27,6 @@ func SplitAddress(backend entity.Backend) (cluster.AddressConfiguration, error) func LogFields(conn *etp.Conn) []log.Field { return []log.Field{ log.String("moduleName", ModuleName(conn)), - log.String("connId", strconv.FormatUint(conn.Id(), 10)), + log.String("connId", conn.Id()), } } diff --git a/middlewares/ws_log.go b/middlewares/ws_log.go index 9a7d766..685ad8e 100644 --- a/middlewares/ws_log.go +++ b/middlewares/ws_log.go @@ -2,9 +2,9 @@ package middlewares import ( "context" + "github.com/txix-open/etp/v4" - "github.com/txix-open/etp/v3" - "github.com/txix-open/etp/v3/msg" + "github.com/txix-open/etp/v4/msg" "github.com/txix-open/isp-kit/log" "isp-config-service/helpers" ) diff --git a/migrations/20241101112923_recreate_backends.sql b/migrations/20241101112923_recreate_backends.sql new file mode 100644 index 0000000..6da81d5 --- /dev/null +++ b/migrations/20241101112923_recreate_backends.sql @@ -0,0 +1,23 @@ +-- +goose Up +drop table isp_config_service__backend; + +create table isp_config_service__backend +( + ws_connection_id text not null, + module_id text not null, + address text not null, + version text not null, + lib_version text not null, + module_name text not null, + config_service_node_id text not null, + endpoints blob not null, + required_modules blob not null, + created_at integer not null default (unixepoch()), + primary key (ws_connection_id), + foreign key (module_id) references isp_config_service__module (id) on delete cascade on update cascade +); + +create index IX_isp_config_service__backend__module_id on isp_config_service__backend (module_id); +create index IX_isp_config_service__backend__node_id on isp_config_service__backend(config_service_node_id); + +-- +goose Down diff --git a/repository/backend.go b/repository/backend.go index 9c90cee..ac191da 100644 --- a/repository/backend.go +++ b/repository/backend.go @@ -21,21 +21,17 @@ func NewBackend(db db.DB) Backend { } } -func (r Backend) Upsert(ctx context.Context, backend entity.Backend) error { - ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.Upsert") +func (r Backend) Insert(ctx context.Context, backend entity.Backend) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.Insert") query, args, err := squirrel.Insert(Table("backend")). - Columns("module_id", "address", - "version", "lib_version", "module_name", + Columns("ws_connection_id", "module_id", "address", + "version", "lib_version", "module_name", "config_service_node_id", "endpoints", "required_modules"). - Values(backend.ModuleId, backend.Address, - backend.Version, backend.LibVersion, backend.ModuleName, + Values(backend.WsConnectionId, backend.ModuleId, backend.Address, + backend.Version, backend.LibVersion, backend.ModuleName, backend.ConfigServiceNodeId, backend.Endpoints, backend.RequiredModules, - ).Suffix(`on conflict (module_id, address) do update - set version = excluded.version, lib_version = excluded.lib_version, module_name = excluded.module_name, - endpoints = excluded.endpoints, required_modules = excluded.required_modules, - updated_at = unixepoch() - `). + ). ToSql() if err != nil { return errors.WithMessage(err, "build query") @@ -49,13 +45,12 @@ func (r Backend) Upsert(ctx context.Context, backend entity.Backend) error { return nil } -func (r Backend) Delete(ctx context.Context, moduleId string, address string) error { - ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.Delete") +func (r Backend) DeleteByWsConnectionIds(ctx context.Context, wsConnectionIds []string) error { + ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.DeleteByWsConnectionIds") query, args, err := squirrel.Delete(Table("backend")). Where(squirrel.Eq{ - "module_id": moduleId, - "address": address, + "ws_connection_id": wsConnectionIds, }).ToSql() if err != nil { return errors.WithMessage(err, "build query") @@ -69,6 +64,52 @@ func (r Backend) Delete(ctx context.Context, moduleId string, address string) er return nil } +func (r Backend) DeleteByConfigServiceNodeId(ctx context.Context, configServiceNodeId string) (int, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.DeleteByConfigServiceNodeId") + + query, args, err := squirrel.Delete(Table("backend")). + Where(squirrel.Eq{ + "config_service_node_id": configServiceNodeId, + }).ToSql() + if err != nil { + return 0, errors.WithMessage(err, "build query") + } + + result, err := r.db.Exec(ctx, query, args...) + if err != nil { + return 0, errors.WithMessagef(err, "exec: %s", query) + } + + deleted, err := result.RowsAffected() + if err != nil { + return 0, errors.WithMessage(err, "rows affected") + } + + return int(deleted), nil +} + +func (r Backend) GetByConfigServiceNodeId(ctx context.Context, configServiceNodeId string) ([]entity.Backend, error) { + ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.GetByConfigServiceNodeId") + + query, args, err := squirrel.Select("*"). + From(Table("backend")). + Where(squirrel.Eq{ + "config_service_node_id": configServiceNodeId, + }).OrderBy("created_at desc"). + ToSql() + if err != nil { + return nil, errors.WithMessage(err, "build query") + } + + result := make([]entity.Backend, 0) + err = r.db.Select(ctx, &result, query, args...) + if err != nil { + return nil, errors.WithMessagef(err, "select: %s", query) + } + + return result, nil +} + func (r Backend) All(ctx context.Context) ([]entity.Backend, error) { ctx = sql_metrics.OperationLabelToContext(ctx, "Backend.All") diff --git a/routes/routes.go b/routes/routes.go index 2ccbc16..40c6368 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -1,10 +1,10 @@ package routes import ( + "github.com/txix-open/etp/v4" "isp-config-service/conf" "net/http" - "github.com/txix-open/etp/v3" "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/grpc" "github.com/txix-open/isp-kit/grpc/endpoint" diff --git a/service/api/module.go b/service/api/module.go index 8e5c260..40c5f85 100644 --- a/service/api/module.go +++ b/service/api/module.go @@ -158,7 +158,7 @@ func (s Module) backendToDto(backend entity.Backend) (domain.Connection, error) Port: addr.Port, }, Endpoints: endpointsDescriptors, - EstablishedAt: time.Time(backend.UpdatedAt), + EstablishedAt: time.Time(backend.CreatedAt), } return conn, nil diff --git a/service/module/backend/backend.go b/service/module/backend/backend.go new file mode 100644 index 0000000..479d15d --- /dev/null +++ b/service/module/backend/backend.go @@ -0,0 +1,156 @@ +package backend + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "github.com/txix-open/etp/v4" + "github.com/txix-open/isp-kit/cluster" + "github.com/txix-open/isp-kit/log" + "isp-config-service/entity" + "isp-config-service/entity/xtypes" +) + +type Repo interface { + Insert(ctx context.Context, backend entity.Backend) error + DeleteByWsConnectionIds(ctx context.Context, wsConnIds []string) error + GetByConfigServiceNodeId(ctx context.Context, configServiceNodeId string) ([]entity.Backend, error) + DeleteByConfigServiceNodeId(ctx context.Context, configServiceNodeId string) (int, error) +} + +type EventRepo interface { + Insert(ctx context.Context, event entity.Event) error +} + +type Backend struct { + backendRepo Repo + eventRepo EventRepo + nodeId string + rooms *etp.Rooms + logger log.Logger +} + +func NewBackend( + backendRepo Repo, + eventRepo EventRepo, + nodeId string, + rooms *etp.Rooms, + logger log.Logger, +) Backend { + return Backend{ + backendRepo: backendRepo, + eventRepo: eventRepo, + nodeId: nodeId, + rooms: rooms, + logger: logger, + } +} + +func (s Backend) Connect( + ctx context.Context, + connId string, + moduleId string, + declaration cluster.BackendDeclaration, +) (*entity.Backend, error) { + backend := entity.Backend{ + WsConnectionId: connId, + ModuleId: moduleId, + Address: fmt.Sprintf("%s:%s", declaration.Address.IP, declaration.Address.Port), + Version: declaration.Version, + LibVersion: declaration.LibVersion, + ModuleName: declaration.ModuleName, + ConfigServiceNodeId: s.nodeId, + Endpoints: xtypes.Json[[]cluster.EndpointDescriptor]{Value: declaration.Endpoints}, + RequiredModules: xtypes.Json[[]cluster.ModuleDependency]{Value: declaration.RequiredModules}, + CreatedAt: xtypes.Time{}, + } + err := s.backendRepo.Insert(ctx, backend) + if err != nil { + return nil, errors.WithMessage(err, "upsert backend in store") + } + + event := entity.NewEvent(entity.EventPayload{ + ModuleReady: &entity.PayloadModuleReady{ + ModuleId: moduleId, + }, + }) + err = s.eventRepo.Insert(ctx, event) + if err != nil { + return nil, errors.WithMessage(err, "insert event in store") + } + + return &backend, nil +} + +func (s Backend) Disconnect(ctx context.Context, backend entity.Backend) error { + err := s.backendRepo.DeleteByWsConnectionIds(ctx, []string{backend.WsConnectionId}) + if err != nil { + return errors.WithMessage(err, "delete backend in store") + } + + err = s.insertDisconnectEvent(ctx, backend.ModuleId) + if err != nil { + return errors.WithMessage(err, "insert disconnect event") + } + + return nil +} + +func (s Backend) ClearPhantomBackends(ctx context.Context) (int, error) { + backends, err := s.backendRepo.GetByConfigServiceNodeId(ctx, s.nodeId) + if err != nil { + return 0, errors.WithMessagef(err, "get by config service node id = '%s'", s.nodeId) + } + + toDelete := make([]string, 0) + modulesToDisconnect := make(map[string]bool) + for _, backend := range backends { + _, ok := s.rooms.Get(backend.WsConnectionId) + if ok { + continue + } + + toDelete = append(toDelete, backend.WsConnectionId) + modulesToDisconnect[backend.ModuleId] = true + } + + if len(toDelete) == 0 { + return 0, nil + } + + err = s.backendRepo.DeleteByWsConnectionIds(ctx, toDelete) + if err != nil { + return 0, errors.WithMessage(err, "delete backends by ws connection ids") + } + + for moduleId := range modulesToDisconnect { + err := s.insertDisconnectEvent(ctx, moduleId) + if err != nil { + return 0, errors.WithMessage(err, "insert disconnect event") + } + } + + return len(toDelete), nil +} + +func (s Backend) DeleteOwnBackends(ctx context.Context) { + deleted, err := s.backendRepo.DeleteByConfigServiceNodeId(ctx, s.nodeId) + if err != nil { + s.logger.Error(ctx, errors.WithMessagef(err, "delete by config service node id = '%s'", s.nodeId)) + return + } + s.logger.Info(ctx, fmt.Sprintf("delete %d own backends", deleted)) +} + +func (s Backend) insertDisconnectEvent(ctx context.Context, moduleId string) error { + event := entity.NewEvent(entity.EventPayload{ + ModuleDisconnected: &entity.PayloadModuleDisconnected{ + ModuleId: moduleId, + }, + }) + err := s.eventRepo.Insert(ctx, event) + if err != nil { + return errors.WithMessage(err, "insert event in store") + } + return nil +} diff --git a/service/module/backend/cleaner.go b/service/module/backend/cleaner.go new file mode 100644 index 0000000..98a668d --- /dev/null +++ b/service/module/backend/cleaner.go @@ -0,0 +1,39 @@ +package backend + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "github.com/txix-open/isp-kit/log" +) + +type Service interface { + ClearPhantomBackends(ctx context.Context) (int, error) +} + +type Cleaner struct { + service Service + logger log.Logger +} + +func NewCleaner( + service Service, + logger log.Logger, +) Cleaner { + return Cleaner{ + service: service, + logger: logger, + } +} + +func (c Cleaner) Do(ctx context.Context) { + ctx = log.ToContext(ctx, log.String("worker", "phantomBackendCleaner")) + + deleted, err := c.service.ClearPhantomBackends(ctx) + if err != nil { + c.logger.Error(ctx, errors.WithMessage(err, "clear phantom backends")) + return + } + + c.logger.Debug(ctx, fmt.Sprintf("delete %d phantom backends", deleted)) +} diff --git a/service/module/emitter.go b/service/module/emitter.go index 3a0d858..e5c84ef 100644 --- a/service/module/emitter.go +++ b/service/module/emitter.go @@ -2,11 +2,11 @@ package module import ( "context" + "github.com/txix-open/etp/v4" "strings" "time" "github.com/pkg/errors" - "github.com/txix-open/etp/v3" "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/log" "isp-config-service/helpers" @@ -51,7 +51,7 @@ func (s Emitter) Emit( if err != nil { err := errors.WithMessagef( err, - "emit event '%s', to %s module, connId: %d", + "emit event '%s', to %s module, connId: %s", event, helpers.ModuleName(conn), conn.Id(), ) s.logger.Error(ctx, err) diff --git a/service/module/service.go b/service/module/service.go index 0e3ed2e..115d46c 100644 --- a/service/module/service.go +++ b/service/module/service.go @@ -4,12 +4,10 @@ import ( "context" "crypto/sha256" "encoding/hex" - "fmt" - "github.com/google/uuid" "github.com/pkg/errors" - "github.com/txix-open/etp/v3" - "github.com/txix-open/etp/v3/store" + "github.com/txix-open/etp/v4" + "github.com/txix-open/etp/v4/store" "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/log" "isp-config-service/entity" @@ -30,13 +28,9 @@ type Repo interface { ) error } -type BackendRepo interface { - Upsert(ctx context.Context, backend entity.Backend) error - Delete(ctx context.Context, moduleId string, address string) error -} - -type EventRepo interface { - Insert(ctx context.Context, event entity.Event) error +type BackendService interface { + Connect(ctx context.Context, connId string, moduleId string, declaration cluster.BackendDeclaration) (*entity.Backend, error) + Disconnect(ctx context.Context, backend entity.Backend) error } type ConfigSchemaRepo interface { @@ -56,8 +50,7 @@ type SubscriptionService interface { type Service struct { moduleRepo Repo - backendRepo BackendRepo - eventRepo EventRepo + backendService BackendService configRepo ConfigRepo configSchemaRepo ConfigSchemaRepo subscriptionService SubscriptionService @@ -67,8 +60,7 @@ type Service struct { func NewService( moduleRepo Repo, - backendRepo BackendRepo, - eventRepo EventRepo, + backendService BackendService, configRepo ConfigRepo, configSchemaRepo ConfigSchemaRepo, subscriptionService SubscriptionService, @@ -77,8 +69,7 @@ func NewService( ) Service { return Service{ moduleRepo: moduleRepo, - backendRepo: backendRepo, - eventRepo: eventRepo, + backendService: backendService, configRepo: configRepo, configSchemaRepo: configSchemaRepo, subscriptionService: subscriptionService, @@ -136,19 +127,9 @@ func (s Service) OnDisconnect( backend, _ := store.Get[entity.Backend](conn.Data(), backendKey) if backend.ModuleId != "" { - err = s.backendRepo.Delete(ctx, backend.ModuleId, backend.Address) - if err != nil { - return errors.WithMessage(err, "delete backend in store") - } - - event := entity.NewEvent(entity.EventPayload{ - ModuleDisconnected: &entity.PayloadModuleDisconnected{ - ModuleId: moduleId, - }, - }) - err = s.eventRepo.Insert(ctx, event) + err := s.backendService.Disconnect(ctx, backend) if err != nil { - return errors.WithMessage(err, "insert event in store") + return errors.WithMessage(err, "disconnect") } } @@ -173,31 +154,12 @@ func (s Service) OnModuleReady( return errors.WithMessage(err, "resolve module id") } - backend := entity.Backend{ - ModuleId: moduleId, - Address: fmt.Sprintf("%s:%s", declaration.Address.IP, declaration.Address.Port), - Version: declaration.Version, - LibVersion: declaration.LibVersion, - ModuleName: declaration.ModuleName, - Endpoints: xtypes.Json[[]cluster.EndpointDescriptor]{Value: declaration.Endpoints}, - RequiredModules: xtypes.Json[[]cluster.ModuleDependency]{Value: declaration.RequiredModules}, - } - err = s.backendRepo.Upsert(ctx, backend) + backend, err := s.backendService.Connect(ctx, conn.Id(), moduleId, declaration) if err != nil { - return errors.WithMessage(err, "upsert backend in store") + return errors.WithMessage(err, "connect") } - conn.Data().Set(backendKey, backend) - - event := entity.NewEvent(entity.EventPayload{ - ModuleReady: &entity.PayloadModuleReady{ - ModuleId: moduleId, - }, - }) - err = s.eventRepo.Insert(ctx, event) - if err != nil { - return errors.WithMessage(err, "insert event in store") - } + conn.Data().Set(backendKey, *backend) return nil } diff --git a/service/startup/service.go b/service/startup/service.go index b071f46..d75361b 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -7,7 +7,6 @@ import ( "github.com/pkg/errors" "github.com/pressly/goose/v3" - "github.com/txix-open/etp/v3" "github.com/txix-open/isp-kit/app" "github.com/txix-open/isp-kit/bootstrap" "github.com/txix-open/isp-kit/cluster" @@ -18,7 +17,6 @@ import ( "github.com/txix-open/isp-kit/http/httpclix" "github.com/txix-open/isp-kit/log" "github.com/txix-open/isp-kit/observability/sentry" - "github.com/txix-open/isp-kit/worker" "isp-config-service/assembly" "isp-config-service/conf" "isp-config-service/middlewares" @@ -41,9 +39,7 @@ type Service struct { logger log.Logger // initialized in Run - etpSrv *etp.Server - handleEventWorker *worker.Worker - cleanEventWorker *worker.Worker + locatorConfig *assembly.Config } func New(boot *bootstrap.Bootstrap) (*Service, error) { @@ -121,15 +117,13 @@ func (s *Service) Run(ctx context.Context) error { Local: s.cfg, RqliteAddress: s.rqlite.LocalHttpAddr(), } - config := assembly.NewLocator(db, s.rqlite, cfg, s.logger).Config() - s.etpSrv = config.EtpSrv - s.handleEventWorker = config.HandleEventWorker - s.cleanEventWorker = config.CleanEventWorker - s.grpcSrv.Upgrade(config.GrpcMux) - s.httpSrv.Upgrade(config.HttpMux) + s.locatorConfig = assembly.NewLocator(db, s.rqlite, cfg, s.logger).Config() + s.grpcSrv.Upgrade(s.locatorConfig.GrpcMux) + s.httpSrv.Upgrade(s.locatorConfig.HttpMux) - s.handleEventWorker.Run(ctx) - s.cleanEventWorker.Run(ctx) + s.locatorConfig.HandleEventWorker.Run(ctx) + s.locatorConfig.CleanEventWorker.Run(ctx) + s.locatorConfig.CleanPhantomBackendWorker.Run(ctx) go func() { s.logger.Debug(ctx, fmt.Sprintf("starting grpc server on %s", s.boot.BindingAddress)) @@ -174,20 +168,19 @@ func (s *Service) Closers() []app.Closer { return s.httpSrv.Shutdown(ctx) }), app.CloserFunc(func() error { - if s.handleEventWorker != nil { - s.handleEventWorker.Shutdown() - } - if s.cleanEventWorker != nil { - s.cleanEventWorker.Shutdown() - } - return nil - }), - app.CloserFunc(func() error { - if s.etpSrv == nil { + if s.locatorConfig == nil { return nil } - s.etpSrv.OnDisconnect(nil) - s.etpSrv.Shutdown() + + s.locatorConfig.HandleEventWorker.Shutdown() + s.locatorConfig.CleanEventWorker.Shutdown() + s.locatorConfig.CleanPhantomBackendWorker.Shutdown() + + s.locatorConfig.EtpSrv.OnDisconnect(nil) + s.locatorConfig.EtpSrv.Shutdown() + + s.locatorConfig.OwnBackendsCleaner.DeleteOwnBackends(context.Background()) + return nil }), s.rqlite, diff --git a/service/subscription/service.go b/service/subscription/service.go index e7a3361..23287fa 100644 --- a/service/subscription/service.go +++ b/service/subscription/service.go @@ -2,9 +2,9 @@ package subscription import ( "context" + "github.com/txix-open/etp/v4" "github.com/pkg/errors" - "github.com/txix-open/etp/v3" "github.com/txix-open/isp-kit/cluster" "github.com/txix-open/isp-kit/json" "github.com/txix-open/isp-kit/log" From 2db855d3604647dea4ce6cb600dc848847c65789 Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Fri, 1 Nov 2024 18:58:16 +0300 Subject: [PATCH 39/40] Pass backend connection id --- domain/response.go | 1 + service/api/module.go | 1 + 2 files changed, 2 insertions(+) diff --git a/domain/response.go b/domain/response.go index 80db8bf..90faa66 100644 --- a/domain/response.go +++ b/domain/response.go @@ -22,6 +22,7 @@ type ModuleInfo struct { } type Connection struct { + Id string ModuleName string LibVersion string Version string diff --git a/service/api/module.go b/service/api/module.go index 40c5f85..596c43f 100644 --- a/service/api/module.go +++ b/service/api/module.go @@ -150,6 +150,7 @@ func (s Module) backendToDto(backend entity.Backend) (domain.Connection, error) }) conn := domain.Connection{ + Id: backend.WsConnectionId, ModuleName: backend.ModuleName, LibVersion: backend.LibVersion, Version: backend.Version, From 120abaf6dd922b26fd378d419aa2beeb6a31d45f Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Fri, 1 Nov 2024 19:04:31 +0300 Subject: [PATCH 40/40] Update swagger --- docs/swagger.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index b87c30c..45581b8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -73,8 +73,12 @@ definitions: type: array establishedAt: type: string + id: + type: string libVersion: type: string + moduleName: + type: string version: type: string type: object