From 4e7571f164274a5e3f78f183d011768e8536fd1f Mon Sep 17 00:00:00 2001 From: lleohao Date: Sat, 9 Nov 2019 16:03:00 +0800 Subject: [PATCH 01/23] SUBMARINE-257. Submarine web user manager page with angular ### What is this PR for? Submarine web, add user manager page with angular ### What type of PR is it? [Feature] ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-257 ### How should this be tested? https://travis-ci.org/lleohao/hadoop-submarine/builds/609567077 ### Screenshots (if appropriate) 1. user list ![image](https://user-images.githubusercontent.com/3677382/68528827-cc5b3900-0332-11ea-9094-081ec7e12c8c.png) 2. add user ![image](https://user-images.githubusercontent.com/3677382/68528836-e1d06300-0332-11ea-93f5-61ee42a2a53c.png) 3. view user detail ![image](https://user-images.githubusercontent.com/3677382/68528857-088e9980-0333-11ea-9747-fc992451983e.png) 4. edit user ![image](https://user-images.githubusercontent.com/3677382/68528859-0debe400-0333-11ea-8d23-25e0535a2277.png) 5. change user password ![image](https://user-images.githubusercontent.com/3677382/68528851-fca2d780-0332-11ea-82fc-b2d42e8f5eb9.png) 6. delete user ![image](https://user-images.githubusercontent.com/3677382/68528848-f7de2380-0332-11ea-939c-b4d9a751158e.png) 7. Log out ![image](https://user-images.githubusercontent.com/3677382/68528854-02002200-0333-11ea-9269-bdfea5664c9a.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: lleohao Closes #89 from lleohao/SUBMARINE-257 and squashes the following commits: 66e7c44 [lleohao] Merge branch 'master' into SUBMARINE-257 d6ddf35 [lleohao] [SUBMARINE-257] ci: fix build ng error af4dff5 [lleohao] [SUBMARINE-257] test: fix e2e test case 79663bb [lleohao] [SUBMARINE-257] ci: add unit test b977f89 [lleohao] [SUBMARINE-257] feat: add manager user feature 3d7055e [lleohao] [SUBMARINE-257] feat: add page header and logout feature 028b399 [lleohao] [SUBMARINE-257] feat: add sidebar and logo 20b96b6 [lleohao] [SUBMARINE-257] feat: init workbench module --- .gitignore | 5 + .travis.yml | 25 +- .../apache/submarine/rest/LoginRestApi.java | 10 +- .../workbench-web-ng/angular.json | 5 +- .../e2e/protractor-ci.conf.js | 33 +++ .../workbench-web-ng/e2e/src/app.e2e-spec.ts | 4 +- .../workbench-web-ng/e2e/src/app.po.ts | 2 +- .../workbench-web-ng/karma.conf.js | 6 + .../workbench-web-ng/package.json | 2 + submarine-workbench/workbench-web-ng/pom.xml | 1 + .../src/app/app-routing.module.ts | 6 +- .../src/app/app.component.html | 52 ---- .../src/app/app.component.scss | 77 ------ .../src/app/app.component.spec.ts | 16 +- .../workbench-web-ng/src/app/app.module.ts | 2 +- .../components.module.ts} | 13 +- .../page-layout/page-layout.component.html | 36 +++ .../page-layout/page-layout.component.scss | 23 ++ .../page-layout/page-layout.component.ts | 37 +++ .../src/app/interfaces/{user.ts => action.ts} | 15 +- .../src/app/interfaces/base-entity.ts | 34 +++ .../src/app/interfaces/permission.ts | 31 ++- .../src/app/interfaces/public-api.ts | 4 +- .../src/app/interfaces/rest.ts | 31 +++ .../src/app/interfaces/role.ts | 25 +- .../src/app/interfaces/sys-dept-select.ts | 34 +++ .../src/app/interfaces/sys-dict-item.ts | 40 +++ .../src/app/interfaces/sys-user.ts | 37 +++ .../src/app/interfaces/user-info.ts | 56 ++++ .../app/pages/user/login/login.component.ts | 11 +- .../src/app/pages/user/user.component.ts | 1 + .../data-dict/data-dict.component.html} | 2 +- .../data-dict/data-dict.component.scss} | 0 .../manager/data-dict/data-dict.component.ts} | 14 +- .../manager/manager-routing.module.ts | 7 +- .../manager/manager.component.html | 9 +- .../manager/manager.component.scss} | 0 .../workbench/manager/manager.component.ts | 57 ++++ .../pages/workbench/manager/manager.module.ts | 37 +++ .../user-drawer/user-drawer.component.html | 206 +++++++++++++++ .../user-drawer/user-drawer.component.scss | 29 +++ .../user-drawer/user-drawer.component.ts | 243 ++++++++++++++++++ .../user-password-modal.component.html | 62 +++++ .../user-password-modal.component.scss | 19 ++ .../user-password-modal.component.ts | 83 ++++++ .../manager/user/user.component.html | 131 ++++++++++ .../manager/user/user.component.scss} | 19 +- .../workbench/manager/user/user.component.ts | 172 +++++++++++++ .../workbench/workbench-routing.module.ts | 46 ++++ .../pages/workbench/workbench.component.html | 82 ++++++ .../pages/workbench/workbench.component.scss | 107 ++++++++ .../pages/workbench/workbench.component.ts | 88 +++++++ .../app/pages/workbench/workbench.module.ts | 32 +++ .../src/app/services/auth.service.ts | 56 ++-- .../src/app/services/base-api.service.ts | 35 +++ .../src/app/services/department.service.ts | 48 ++++ .../src/app/services/public-api.ts | 5 + .../src/app/services/system-utils.service.ts | 95 +++++++ .../src/app/services/user.service.ts | 139 ++++++++++ .../workbench-web-ng/src/assets/logo.png | Bin 0 -> 16547 bytes .../workbench-web-ng/src/main.ts | 3 +- .../workbench-web-ng/src/polyfills.ts | 3 +- .../workbench-web-ng/src/test.ts | 10 +- 63 files changed, 2261 insertions(+), 252 deletions(-) create mode 100644 submarine-workbench/workbench-web-ng/e2e/protractor-ci.conf.js rename submarine-workbench/workbench-web-ng/src/app/{pages/manager/manager.module.ts => components/components.module.ts} (74%) create mode 100644 submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.html create mode 100644 submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.scss create mode 100644 submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.ts rename submarine-workbench/workbench-web-ng/src/app/interfaces/{user.ts => action.ts} (82%) create mode 100644 submarine-workbench/workbench-web-ng/src/app/interfaces/base-entity.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/interfaces/sys-dept-select.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/interfaces/sys-dict-item.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/interfaces/sys-user.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/interfaces/user-info.ts rename submarine-workbench/workbench-web-ng/src/app/pages/{manager/user/user.component.html => workbench/manager/data-dict/data-dict.component.html} (97%) rename submarine-workbench/workbench-web-ng/src/app/pages/{manager/manager.component.scss => workbench/manager/data-dict/data-dict.component.scss} (100%) rename submarine-workbench/workbench-web-ng/src/app/pages/{manager/manager.component.ts => workbench/manager/data-dict/data-dict.component.ts} (80%) rename submarine-workbench/workbench-web-ng/src/app/pages/{ => workbench}/manager/manager-routing.module.ts (86%) rename submarine-workbench/workbench-web-ng/src/app/pages/{ => workbench}/manager/manager.component.html (79%) rename submarine-workbench/workbench-web-ng/src/app/pages/{manager/user/user.component.scss => workbench/manager/manager.component.scss} (100%) create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.module.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.html create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.scss create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.html create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.scss create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.html rename submarine-workbench/workbench-web-ng/src/app/pages/{manager/user/user.component.ts => workbench/manager/user/user.component.scss} (75%) create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench-routing.module.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.html create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.scss create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.module.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/services/department.service.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/services/system-utils.service.ts create mode 100644 submarine-workbench/workbench-web-ng/src/app/services/user.service.ts create mode 100644 submarine-workbench/workbench-web-ng/src/assets/logo.png diff --git a/.gitignore b/.gitignore index a514d00a0e..521d78c2e7 100644 --- a/.gitignore +++ b/.gitignore @@ -67,4 +67,9 @@ submarine-workbench/workbench-web/.env.*.local submarine-workbench/workbench-web/npm-debug.log* submarine-workbench/workbench-web/yarn-debug.log* submarine-workbench/workbench-web/yarn-error.log* +submarine-workbench/workbench-web-ng/node +submarine-workbench/workbench-web-ng/node_modules +submarine-workbench/workbench-web-ng/dist +submarine-workbench/workbench-web-ng/package-lock.json +submarine-workbench/workbench-web-ng/npm-debug.log* submarine-test/e2e/Driver diff --git a/.travis.yml b/.travis.yml index 313743eaf1..1b25f562d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,8 +22,10 @@ cache: apt: true directories: - ${HOME}/.m2 - - submarine-workbench/workbench-web/node - - submarine-workbench/workbench-web/node_modules + - submarine-workbench/workbench-web/node + - submarine-workbench/workbench-web/node_modules + - submarine-workbench/workbench-web-ng/node + - submarine-workbench/workbench-web-ng/node_modules addons: apt: @@ -139,6 +141,25 @@ matrix: dist: xenial env: NAME="Test submarine distribution" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="" TEST_MODULES="" TEST_PROJECTS="" + # Test submarine web-ng + - language: node_js + node_js: + - "10" + before_install: + - cd submarine-workbench/workbench-web-ng + before_script: + npm install + addons: + apt: + sources: + - google-chrome + packages: + - google-chrome-stable + script: + - npm run test -- --no-watch --no-progress --browsers=ChromeHeadlessCI + - npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js + env: NAME="Build workbench-web-ng" + install: - mvn --version - echo "[$NAME] > mvn $BUILD_FLAG $MODULES $PROFILE -B" diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/LoginRestApi.java b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/LoginRestApi.java index df21f29e76..f0b89c7211 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/LoginRestApi.java +++ b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/LoginRestApi.java @@ -59,11 +59,12 @@ public Response login(String loginParams) { try (SqlSession sqlSession = MyBatisUtil.getSqlSession()) { SysUserMapper sysUserMapper = sqlSession.getMapper(SysUserMapper.class); sysUser = sysUserMapper.login(mapParams); + + sysUser.setToken("mock_token"); } catch (Exception e) { LOG.error(e.getMessage(), e); return new JsonResponse.Builder<>(Response.Status.OK).success(false).build(); } - sysUser.setToken("mock_token"); return new JsonResponse.Builder(Response.Status.OK).success(true).result(sysUser).build(); } @@ -76,4 +77,11 @@ public Response step() { return new JsonResponse.Builder(Response.Status.OK).success(true).result(data).build(); } + + @POST + @Path("/logout") + @SubmarineApi + public Response logout() { + return new JsonResponse.Builder(Response.Status.OK).success(true).result(true).build(); + } } diff --git a/submarine-workbench/workbench-web-ng/angular.json b/submarine-workbench/workbench-web-ng/angular.json index c47d29ddb1..b8ea29d7ec 100644 --- a/submarine-workbench/workbench-web-ng/angular.json +++ b/submarine-workbench/workbench-web-ng/angular.json @@ -101,7 +101,8 @@ "styles": [ "src/styles.scss" ], - "scripts": [] + "scripts": [], + "codeCoverage": true } }, "lint": { @@ -133,4 +134,4 @@ } }, "defaultProject": "workbench-web-ng" -} \ No newline at end of file +} diff --git a/submarine-workbench/workbench-web-ng/e2e/protractor-ci.conf.js b/submarine-workbench/workbench-web-ng/e2e/protractor-ci.conf.js new file mode 100644 index 0000000000..11abcc4b92 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/e2e/protractor-ci.conf.js @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @ts-check +// Protractor configuration file, see link for more information +// https://github.com/angular/protractor/blob/master/lib/config.ts + +const config = require('./protractor.conf').config; + +config.capabilities = { + browserName: 'chrome', + chromeOptions: { + args: ['--headless', '--no-sandbox'] + } +}; + +exports.config = config; diff --git a/submarine-workbench/workbench-web-ng/e2e/src/app.e2e-spec.ts b/submarine-workbench/workbench-web-ng/e2e/src/app.e2e-spec.ts index 5b0d8f69ef..d91a449404 100644 --- a/submarine-workbench/workbench-web-ng/e2e/src/app.e2e-spec.ts +++ b/submarine-workbench/workbench-web-ng/e2e/src/app.e2e-spec.ts @@ -27,9 +27,9 @@ describe('workspace-project App', () => { page = new AppPage(); }); - it('should display welcome message', () => { + it('should display submarine', () => { page.navigateTo(); - expect(page.getTitleText()).toEqual('workbench-web-ng app is running!'); + expect(page.getTitleText()).toEqual('Submarine'); }); afterEach(async () => { diff --git a/submarine-workbench/workbench-web-ng/e2e/src/app.po.ts b/submarine-workbench/workbench-web-ng/e2e/src/app.po.ts index 493318b21d..62e9fb6452 100644 --- a/submarine-workbench/workbench-web-ng/e2e/src/app.po.ts +++ b/submarine-workbench/workbench-web-ng/e2e/src/app.po.ts @@ -25,6 +25,6 @@ export class AppPage { } getTitleText() { - return element(by.css('app-root .content span')).getText() as Promise; + return element(by.css('submarine-root submarine-user .title')).getText() as Promise; } } diff --git a/submarine-workbench/workbench-web-ng/karma.conf.js b/submarine-workbench/workbench-web-ng/karma.conf.js index 00f2741fd5..bfa1d61080 100644 --- a/submarine-workbench/workbench-web-ng/karma.conf.js +++ b/submarine-workbench/workbench-web-ng/karma.conf.js @@ -45,6 +45,12 @@ module.exports = function (config) { logLevel: config.LOG_INFO, autoWatch: true, browsers: ['Chrome'], + customLaunchers: { + ChromeHeadlessCI: { + base: 'ChromeHeadless', + flags: ['--no-sandbox'] + } + }, singleRun: false, restartOnFileChange: true }); diff --git a/submarine-workbench/workbench-web-ng/package.json b/submarine-workbench/workbench-web-ng/package.json index c2deab23d1..8fc2e31ced 100644 --- a/submarine-workbench/workbench-web-ng/package.json +++ b/submarine-workbench/workbench-web-ng/package.json @@ -19,6 +19,8 @@ "@angular/platform-browser": "~8.2.9", "@angular/platform-browser-dynamic": "~8.2.9", "@angular/router": "~8.2.9", + "date-fns": "^2.6.0", + "lodash": "^4.17.15", "md5": "^2.2.1", "ng-zorro-antd": "8.1.2", "rxjs": "~6.4.0", diff --git a/submarine-workbench/workbench-web-ng/pom.xml b/submarine-workbench/workbench-web-ng/pom.xml index 081d2d712b..f74883de90 100644 --- a/submarine-workbench/workbench-web-ng/pom.xml +++ b/submarine-workbench/workbench-web-ng/pom.xml @@ -48,6 +48,7 @@ + compile org.apache.rat diff --git a/submarine-workbench/workbench-web-ng/src/app/app-routing.module.ts b/submarine-workbench/workbench-web-ng/src/app/app-routing.module.ts index 6cb4725943..6ca7ea429c 100644 --- a/submarine-workbench/workbench-web-ng/src/app/app-routing.module.ts +++ b/submarine-workbench/workbench-web-ng/src/app/app-routing.module.ts @@ -25,12 +25,12 @@ const routes: Routes = [ { path: '', pathMatch: 'full', - redirectTo: '/manager/user' + redirectTo: 'workbench' }, { - path: 'manager', + path: 'workbench', canActivate: [AuthGuard], - loadChildren: () => import('./pages/manager/manager.module').then(m => m.ManagerModule) + loadChildren: () => import('./pages/workbench/workbench.module').then(m => m.WorkbenchModule) }, { path: 'user', diff --git a/submarine-workbench/workbench-web-ng/src/app/app.component.html b/submarine-workbench/workbench-web-ng/src/app/app.component.html index 6c447c048d..970329bad2 100644 --- a/submarine-workbench/workbench-web-ng/src/app/app.component.html +++ b/submarine-workbench/workbench-web-ng/src/app/app.component.html @@ -19,56 +19,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/submarine-workbench/workbench-web-ng/src/app/app.component.scss b/submarine-workbench/workbench-web-ng/src/app/app.component.scss index 67bafe67fa..a3b64854a4 100644 --- a/submarine-workbench/workbench-web-ng/src/app/app.component.scss +++ b/submarine-workbench/workbench-web-ng/src/app/app.component.scss @@ -17,83 +17,6 @@ * under the License. */ -:host { - display: flex; - text-rendering: optimizeLegibility; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} - .app-layout { height: 100vh; } - -.menu-sidebar { - position: relative; - z-index: 10; - min-height: 100vh; - box-shadow: 2px 0 6px rgba(0,21,41,.35); -} - -.header-trigger { - height: 64px; - padding: 20px 24px; - font-size: 20px; - cursor: pointer; - transition: all .3s,padding 0s; -} - -.trigger:hover { - color: #1890ff; -} - -.sidebar-logo { - position: relative; - height: 64px; - padding-left: 24px; - overflow: hidden; - line-height: 64px; - background: #001529; - transition: all .3s; -} - -.sidebar-logo img { - display: inline-block; - height: 32px; - width: 32px; - vertical-align: middle; -} - -.sidebar-logo h1 { - display: inline-block; - margin: 0 0 0 20px; - color: #fff; - font-weight: 600; - font-size: 14px; - font-family: Avenir,Helvetica Neue,Arial,Helvetica,sans-serif; - vertical-align: middle; -} - -nz-header { - padding: 0; - width: 100%; - z-index: 2; -} - -.app-header { - position: relative; - height: 64px; - padding: 0; - background: #fff; - box-shadow: 0 1px 4px rgba(0,21,41,.08); -} - -nz-content { - margin: 24px; -} - -.inner-content { - padding: 24px; - background: #fff; - height: 100%; -} diff --git a/submarine-workbench/workbench-web-ng/src/app/app.component.spec.ts b/submarine-workbench/workbench-web-ng/src/app/app.component.spec.ts index e59a500ba2..c26eb91e19 100644 --- a/submarine-workbench/workbench-web-ng/src/app/app.component.spec.ts +++ b/submarine-workbench/workbench-web-ng/src/app/app.component.spec.ts @@ -19,12 +19,13 @@ import { async, TestBed } from '@angular/core/testing'; import { RouterTestingModule } from '@angular/router/testing'; +import { NgZorroAntdModule } from 'ng-zorro-antd'; import { AppComponent } from './app.component'; describe('AppComponent', () => { beforeEach(async(() => { TestBed.configureTestingModule({ - imports: [RouterTestingModule], + imports: [RouterTestingModule, NgZorroAntdModule], declarations: [AppComponent] }).compileComponents(); })); @@ -34,17 +35,4 @@ describe('AppComponent', () => { const app = fixture.debugElement.componentInstance; expect(app).toBeTruthy(); }); - - it(`should have as title 'workbench-web-ng'`, () => { - const fixture = TestBed.createComponent(AppComponent); - const app = fixture.debugElement.componentInstance; - expect(app.title).toEqual('workbench-web-ng'); - }); - - it('should render title', () => { - const fixture = TestBed.createComponent(AppComponent); - fixture.detectChanges(); - const compiled = fixture.debugElement.nativeElement; - expect(compiled.querySelector('.content span').textContent).toContain('workbench-web-ng app is running!'); - }); }); diff --git a/submarine-workbench/workbench-web-ng/src/app/app.module.ts b/submarine-workbench/workbench-web-ng/src/app/app.module.ts index 59c9cfdebc..64acdb46e2 100644 --- a/submarine-workbench/workbench-web-ng/src/app/app.module.ts +++ b/submarine-workbench/workbench-web-ng/src/app/app.module.ts @@ -25,7 +25,7 @@ import { HttpClientModule } from '@angular/common/http'; import zh from '@angular/common/locales/zh'; import { FormsModule } from '@angular/forms'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; -import { LocalStorageService } from '@submarine/services/local-storage.service'; +import { LocalStorageService } from '@submarine/services'; import { zh_CN, NgZorroAntdModule, NZ_I18N } from 'ng-zorro-antd'; import { AppRoutingModule } from './app-routing.module'; import { AppComponent } from './app.component'; diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.module.ts b/submarine-workbench/workbench-web-ng/src/app/components/components.module.ts similarity index 74% rename from submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.module.ts rename to submarine-workbench/workbench-web-ng/src/app/components/components.module.ts index cfabafeada..bfbd54c19e 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.module.ts +++ b/submarine-workbench/workbench-web-ng/src/app/components/components.module.ts @@ -19,13 +19,14 @@ import { CommonModule } from '@angular/common'; import { NgModule } from '@angular/core'; +import { RouterModule } from '@angular/router'; import { NgZorroAntdModule } from 'ng-zorro-antd'; -import { ManagerRoutingModule } from './manager-routing.module'; -import { ManagerComponent } from './manager.component'; -import { UserComponent } from './user/user.component'; +import { PageLayoutComponent } from './page-layout/page-layout.component'; @NgModule({ - declarations: [UserComponent, ManagerComponent], - imports: [CommonModule, ManagerRoutingModule, NgZorroAntdModule] + declarations: [PageLayoutComponent], + imports: [CommonModule, RouterModule, NgZorroAntdModule], + exports: [PageLayoutComponent] }) -export class ManagerModule {} +export class ComponentsModule { +} diff --git a/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.html b/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.html new file mode 100644 index 0000000000..bf9287dd65 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.html @@ -0,0 +1,36 @@ + + +
+ + + + {{item|titlecase}} + + + +

{{description}}

+
+
+ +
+ +
+
+ diff --git a/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.scss b/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.scss new file mode 100644 index 0000000000..ecea51cf63 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.scss @@ -0,0 +1,23 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +nz-page-header { + margin: -24px -24px 16px; +} + diff --git a/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.ts b/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.ts new file mode 100644 index 0000000000..77a8a2814d --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/components/page-layout/page-layout.component.ts @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Component, Input, OnInit } from '@angular/core'; + +@Component({ + selector: 'submarine-page-layout', + templateUrl: './page-layout.component.html', + styleUrls: ['./page-layout.component.scss'] +}) +export class PageLayoutComponent implements OnInit { + @Input() title: string; + @Input() description: string; + @Input() breadCrumb: string[]; + + constructor() { + } + + ngOnInit() { + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/user.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/action.ts similarity index 82% rename from submarine-workbench/workbench-web-ng/src/app/interfaces/user.ts rename to submarine-workbench/workbench-web-ng/src/app/interfaces/action.ts index c68c4242d3..525ed78c5a 100644 --- a/submarine-workbench/workbench-web-ng/src/app/interfaces/user.ts +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/action.ts @@ -17,15 +17,8 @@ * under the License. */ -import { Role } from './role'; - -export class User { - avatar = ''; - id = ''; - name = ''; - telephone = ''; - username = ''; - role: Role; - roleId: number; - token: string; +export interface Action { + action: string; + defaultCheck: boolean; + describe: string; } diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/base-entity.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/base-entity.ts new file mode 100644 index 0000000000..00dbaa319f --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/base-entity.ts @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export class BaseEntity { + id: string; + createBy: string; + createTime: number; + updateBy: string; + updateTime: number; + + constructor(data: BaseEntity) { + this.id = data.id; + this.createBy = data.createBy; + this.createTime = data.createTime; + this.updateBy = data.updateBy; + this.updateTime = data.updateTime; + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/permission.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/permission.ts index fffef9eb40..349649a547 100644 --- a/submarine-workbench/workbench-web-ng/src/app/interfaces/permission.ts +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/permission.ts @@ -17,19 +17,30 @@ * under the License. */ -export interface PermissionActionEntitySet { +import { Action } from './action'; + +export interface ActionEntitySet { action: string; - defaultChecked: boolean; + defaultCheck: boolean; describe: string; } -export type PermissionAction = PermissionActionEntitySet; - export class Permission { - permissionId = ''; - permissionName = ''; - roleId = ''; - actionList = null; - actionEntitySet: PermissionActionEntitySet[] = []; - actions: PermissionAction[] = []; + roleId: string; + permissionId: string; + permissionName: string; + dataAccess?: any; + actionList?: any; + actions: Action[]; + actionEntitySet: ActionEntitySet[]; + + constructor(permission: Permission) { + this.roleId = permission.roleId; + this.permissionId = permission.permissionId; + this.permissionName = permission.permissionName; + this.dataAccess = permission.dataAccess; + this.actionList = permission.actionList; + this.actions = permission.actions; + this.actionEntitySet = permission.actionEntitySet; + } } diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/public-api.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/public-api.ts index 89585222ee..4123a936fc 100644 --- a/submarine-workbench/workbench-web-ng/src/app/interfaces/public-api.ts +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/public-api.ts @@ -19,5 +19,7 @@ export * from './permission'; export * from './role'; -export * from './user'; +export * from './user-info'; export * from './rest'; +export * from './action'; +export * from './sys-user'; diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/rest.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/rest.ts index dcf39fc54c..61f5d46dfe 100644 --- a/submarine-workbench/workbench-web-ng/src/app/interfaces/rest.ts +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/rest.ts @@ -17,10 +17,41 @@ * under the License. */ +/** + * REST api abstract interface + * + * @example + * ```typescript + * const res = Rest<{userName: string, password: string}>; + * + * // res.result.userName is string + * // res.result.password is string + * ``` + */ export interface Rest { + /** + * request result + */ result: T; + /** + * request http status code + */ code: number; message: string; status: string; success: boolean; } + +/** + * Array result + */ +export interface ListResult { + /** + * result list + */ + records: T[]; + /** + * result list length + */ + total: number; +} diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/role.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/role.ts index f3f8669e23..1464300dd8 100644 --- a/submarine-workbench/workbench-web-ng/src/app/interfaces/role.ts +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/role.ts @@ -19,13 +19,24 @@ import { Permission } from './permission'; -export type RoleId = string; - export class Role { - createTime: number = -1; - creatorId = ''; - describe = ''; - id = ''; - name = ''; + id: string; + name: string; + describe: string; + status: number; + creatorId: string; + createTime: number; + deleted: number; permissions: Permission[]; + + constructor(role: Role) { + this.id = role.id; + this.name = role.name; + this.describe = role.describe; + this.status = role.status; + this.creatorId = role.creatorId; + this.createTime = role.createTime; + this.deleted = role.deleted; + this.permissions = role.permissions.map(permission => new Permission(permission)); + } } diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-dept-select.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-dept-select.ts new file mode 100644 index 0000000000..5929e7bd3e --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-dept-select.ts @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export class SysDeptSelect { + key: string; + value: string; + title: string; + disabled: boolean; + children: SysDeptSelect[]; + + constructor(data: SysDeptSelect) { + this.key = data.key; + this.value = data.value; + this.title = data.title; + this.disabled = data.disabled; + this.children = data.children.map(item => new SysDeptSelect(item)); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-dict-item.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-dict-item.ts new file mode 100644 index 0000000000..dbe56d1598 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-dict-item.ts @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { BaseEntity } from '@submarine/interfaces/base-entity'; + +export class SysDictItem extends BaseEntity { + dictCode: string; + itemCode: string; + itemName: string; + description: string; + sortOrder: number; + deleted: number; + + constructor(data: SysDictItem) { + super(data); + + this.dictCode = data.dictCode; + this.itemCode = data.itemCode; + this.itemName = data.itemName; + this.description = data.description; + this.sortOrder = data.sortOrder; + this.deleted = data.deleted; + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-user.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-user.ts new file mode 100644 index 0000000000..18571e771a --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/sys-user.ts @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { BaseEntity } from './base-entity'; + +export interface SysUser extends BaseEntity { + userName: string; + realName: string; + password: string; + avatar: string; + sex: string; + status: string; + phone: string; + email: string; + deptCode: string; + deptName: string; + roleCode: string; + birthday: number; + deleted: number; + token: string; +} diff --git a/submarine-workbench/workbench-web-ng/src/app/interfaces/user-info.ts b/submarine-workbench/workbench-web-ng/src/app/interfaces/user-info.ts new file mode 100644 index 0000000000..c159497426 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/interfaces/user-info.ts @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Role } from './role'; + +export class UserInfo { + id: string; + name: string; + username: string; + password: string; + avatar: string; + status: number; + telephone: string; + lastLoginIp: string; + lastLoginTime: number; + creatorId: string; + createTime: number; + merchantCode: string; + deleted: number; + roleId: string; + role: Role; + + constructor(res: UserInfo) { + this.id = res.id; + this.name = res.name; + this.username = res.username; + this.password = res.password; + this.avatar = res.avatar; + this.status = res.status; + this.telephone = res.telephone; + this.lastLoginIp = res.lastLoginIp; + this.lastLoginTime = res.lastLoginTime; + this.creatorId = res.creatorId; + this.createTime = res.createTime; + this.merchantCode = res.merchantCode; + this.deleted = res.deleted; + this.roleId = res.roleId; + this.role = new Role(res.role); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/user/login/login.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/user/login/login.component.ts index 9190cea1aa..2158604322 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/user/login/login.component.ts +++ b/submarine-workbench/workbench-web-ng/src/app/pages/user/login/login.component.ts @@ -20,8 +20,8 @@ import { Component, OnInit } from '@angular/core'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { Router } from '@angular/router'; +import { AuthService } from '@submarine/services'; import { NzNotificationService } from 'ng-zorro-antd'; -import { AuthService } from '../../../services'; @Component({ selector: 'submarine-login', @@ -38,7 +38,7 @@ export class LoginComponent implements OnInit { private router: Router ) { if (this.authService.isLoggedIn) { - this.router.navigate(['/manager/user']); + this.router.navigate(['/workbench']); } } @@ -55,7 +55,6 @@ export class LoginComponent implements OnInit { this.loginSuccess(); }, error => { - console.log(error); this.requestFailed(error); } ); @@ -71,11 +70,7 @@ export class LoginComponent implements OnInit { } loginSuccess() { - this.router.navigate(['/manager/user']); - - setTimeout(() => { - this.nzNotificationService.success('Welcome', 'Welcome back'); - }, 1000); + this.router.navigate(['/workbench']); } requestFailed(error: Error) { diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/user/user.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/user/user.component.ts index b38a643d78..c4f6328d43 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/user/user.component.ts +++ b/submarine-workbench/workbench-web-ng/src/app/pages/user/user.component.ts @@ -18,6 +18,7 @@ */ import { Component, OnInit } from '@angular/core'; +import { UserService } from '@submarine/services'; @Component({ selector: 'submarine-user', diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/manager/user/user.component.html b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/data-dict/data-dict.component.html similarity index 97% rename from submarine-workbench/workbench-web-ng/src/app/pages/manager/user/user.component.html rename to submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/data-dict/data-dict.component.html index c5412cf557..5f8f81fdd9 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/manager/user/user.component.html +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/data-dict/data-dict.component.html @@ -17,4 +17,4 @@ ~ under the License. --> -

user works!

+

data-dict works!

diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.component.scss b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/data-dict/data-dict.component.scss similarity index 100% rename from submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.component.scss rename to submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/data-dict/data-dict.component.scss diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/data-dict/data-dict.component.ts similarity index 80% rename from submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.component.ts rename to submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/data-dict/data-dict.component.ts index efa6503968..3dde811fe9 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.component.ts +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/data-dict/data-dict.component.ts @@ -20,12 +20,14 @@ import { Component, OnInit } from '@angular/core'; @Component({ - selector: 'submarine-manager', - templateUrl: './manager.component.html', - styleUrls: ['./manager.component.scss'] + selector: 'submarine-data-dict', + templateUrl: './data-dict.component.html', + styleUrls: ['./data-dict.component.scss'] }) -export class ManagerComponent implements OnInit { - constructor() {} +export class DataDictComponent implements OnInit { + constructor() { + } - ngOnInit() {} + ngOnInit() { + } } diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager-routing.module.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager-routing.module.ts similarity index 86% rename from submarine-workbench/workbench-web-ng/src/app/pages/manager/manager-routing.module.ts rename to submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager-routing.module.ts index 97c775f132..52757dc541 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager-routing.module.ts +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager-routing.module.ts @@ -19,6 +19,7 @@ import { NgModule } from '@angular/core'; import { RouterModule, Routes } from '@angular/router'; +import { DataDictComponent } from '@submarine/pages/workbench/manager/data-dict/data-dict.component'; import { ManagerComponent } from './manager.component'; import { UserComponent } from './user/user.component'; @@ -30,11 +31,15 @@ const routes: Routes = [ { path: '', pathMatch: 'full', - redirectTo: '/manager/user' + redirectTo: 'user' }, { path: 'user', component: UserComponent + }, + { + path: 'data-dict', + component: DataDictComponent } ] } diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.component.html b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.html similarity index 79% rename from submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.component.html rename to submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.html index 4f806ffbe7..598d7dba22 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/manager/manager.component.html +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.html @@ -17,5 +17,10 @@ ~ under the License. --> -

manager works!

- + + + diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/manager/user/user.component.scss b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.scss similarity index 100% rename from submarine-workbench/workbench-web-ng/src/app/pages/manager/user/user.component.scss rename to submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.scss diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.ts new file mode 100644 index 0000000000..15f42c56f3 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.component.ts @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Location, LocationStrategy, PathLocationStrategy } from '@angular/common'; +import { Component, OnInit } from '@angular/core'; +import { ActivatedRoute, NavigationEnd, Router } from '@angular/router'; +import _ from 'lodash'; + +interface HeaderInfo { + title: string; + description: string; + breadCrumb: string[]; +} + +@Component({ + selector: 'submarine-manager', + templateUrl: './manager.component.html', + styleUrls: ['./manager.component.scss'], + providers: [Location, { provide: LocationStrategy, useClass: PathLocationStrategy }] +}) +export class ManagerComponent implements OnInit { + private headerInfo: { [key: string]: HeaderInfo } = { + user: { + title: 'user', + description: 'You can check the user, delete the user, lock and unlock the user, etc.', + breadCrumb: ['manager', 'user'] + } + }; + currentHeaderInfo: HeaderInfo; + + constructor(private route: ActivatedRoute, private location: Location, private router: Router) { + this.router.events.subscribe(event => { + if (event instanceof NavigationEnd) { + const lastMatch = _.last(event.urlAfterRedirects.split('/')); + this.currentHeaderInfo = this.headerInfo[lastMatch]; + } + }); + } + + ngOnInit() {} +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.module.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.module.ts new file mode 100644 index 0000000000..94048f526e --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/manager.module.ts @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { CommonModule } from '@angular/common'; +import { NgModule } from '@angular/core'; +import { FormsModule, ReactiveFormsModule } from '@angular/forms'; +import { ComponentsModule } from '@submarine/components/components.module'; +import { NgZorroAntdModule } from 'ng-zorro-antd'; + +import { DataDictComponent } from './data-dict/data-dict.component'; +import { ManagerRoutingModule } from './manager-routing.module'; +import { ManagerComponent } from './manager.component'; +import { UserDrawerComponent } from './user-drawer/user-drawer.component'; +import { UserPasswordModalComponent } from './user-password-modal/user-password-modal.component'; +import { UserComponent } from './user/user.component'; + +@NgModule({ + declarations: [UserComponent, ManagerComponent, DataDictComponent, UserPasswordModalComponent, UserDrawerComponent], + imports: [CommonModule, ManagerRoutingModule, NgZorroAntdModule, ComponentsModule, FormsModule, ReactiveFormsModule] +}) +export class ManagerModule {} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.html b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.html new file mode 100644 index 0000000000..9ae9b64ff6 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.html @@ -0,0 +1,206 @@ + + + +
+ + + + Account Name + + + + + + + Please input account's name! + + + Account name already exist! + + + Error + + + + + + + + + Password + + + + + + + The password consists of 8 digits, uppercase and lowercase letters and special symbols! + + + Pleas input account's password! + + + + + + + + + Confirm + + + + + + + Please confirm your password! + + + Password is inconsistent! + + + + + + + + + Real Name + + + + + + + + + + Department + + + + + + + + + + Birthday + + + + + + + + + + Sex + + + + + + + + + + + + email + + + + + + + Email is incorrect! + + + Email name already exist! + + + + + + + + + Phone + + + + + + + Phone already exist! + + + + + + + + + Status + + + + + + + +
+ + +
diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.scss b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.scss new file mode 100644 index 0000000000..7750c7a524 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.scss @@ -0,0 +1,29 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.footer { + position: absolute; + bottom: 0; + width: 100%; + border-top: 1px solid rgb(232, 232, 232); + padding: 10px 16px; + text-align: right; + left: 0; + background: #fff; +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.ts new file mode 100644 index 0000000000..a480af43e7 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-drawer/user-drawer.component.ts @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Component, EventEmitter, Input, OnChanges, OnInit, Output, SimpleChanges } from '@angular/core'; +import { FormBuilder, FormControl, FormGroup, ValidationErrors, Validators } from '@angular/forms'; +import { SysUser } from '@submarine/interfaces'; +import { SysDeptSelect } from '@submarine/interfaces/sys-dept-select'; +import { SysDictItem } from '@submarine/interfaces/sys-dict-item'; +import { SystemUtilsService, SysDictCode } from '@submarine/services'; +import { format } from 'date-fns'; +import { zip, Observable, Observer } from 'rxjs'; +import { filter, map, startWith, take } from 'rxjs/operators'; + +@Component({ + selector: 'submarine-user-drawer', + templateUrl: './user-drawer.component.html', + styleUrls: ['./user-drawer.component.scss'] +}) +export class UserDrawerComponent implements OnInit, OnChanges { + @Input() visible: boolean; + @Input() readonly: boolean; + @Input() sysUser: SysUser; + @Input() sysDeptTreeList: SysDeptSelect[]; + @Output() readonly close: EventEmitter = new EventEmitter(); + @Output() readonly submit: EventEmitter> = new EventEmitter>(); + + form: FormGroup; + labelSpan = 5; + controlSpan = 14; + sexDictSelect: SysDictItem[] = []; + statusDictSelect: SysDictItem[] = []; + title = 'Add'; + + constructor(private fb: FormBuilder, private systemUtilsService: SystemUtilsService) { + } + + ngOnInit() { + this.form = this.fb.group( + { + userName: new FormControl('', + { + updateOn: 'blur', + validators: [Validators.required], + asyncValidators: [this.userNameValidator] + } + ), + password: ['', [Validators.required, Validators.pattern(/^(?=.*[a-zA-Z])(?=.*\d)(?=.*[~!@#$%^&*()_+`\-={}:";'<>?,./]).{8,}$/)]], + confirm: ['', this.confirmValidator], + realName: ['', Validators.required], + deptCode: [], + sex: [], + status: [], + birthday: [], + email: new FormControl('', { + updateOn: 'blur', + validators: [Validators.email], + asyncValidators: [this.emailValidator] + }), + phone: new FormControl('', { + updateOn: 'blur', + asyncValidators: [this.phoneValidator] + }) + } + ); + + zip( + this.systemUtilsService.fetchSysDictByCode(SysDictCode.USER_SEX), + this.systemUtilsService.fetchSysDictByCode(SysDictCode.USER_STATUS) + ) + .pipe(map(([item1, item2]) => [item1.records, item2.records])) + .subscribe(([sexDictSelect, statusDictSelect]) => { + this.sexDictSelect = sexDictSelect; + this.statusDictSelect = statusDictSelect; + }); + } + + ngOnChanges(changes: SimpleChanges): void { + if (!changes.visible) { + return; + } + + if (changes.visible.currentValue) { + const sysUser = this.sysUser; + const readOnly = this.readonly; + + this.form.reset({ + userName: sysUser ? sysUser.userName : '', + password: sysUser ? sysUser.password : '', + confirm: sysUser ? sysUser.password : '', + realName: sysUser ? sysUser.realName : '', + deptCode: { + value: sysUser ? sysUser.deptCode : '', + disabled: readOnly + }, + sex: { + value: sysUser ? sysUser.sex : '', + disabled: readOnly + }, + status: { + value: sysUser ? sysUser.status : '', + disabled: readOnly + }, + birthday: { + value: (sysUser && sysUser.birthday) ? new Date(sysUser.birthday) : '', + disabled: readOnly + }, + email: sysUser ? sysUser.email : '', + phone: sysUser ? sysUser.phone : '' + }); + this.title = this.generateTitle(); + } + } + + onClose() { + this.close.emit(); + } + + onSubmit() { + for (const key in this.form.controls) { + this.form.controls[key].markAsDirty(); + this.form.controls[key].updateValueAndValidity(); + } + + this.form.statusChanges.pipe( + startWith(this.form.status), + filter(status => status !== 'PENDING'), + take(1), + map(status => status === 'VALID') + ).subscribe(valid => { + if (valid) { + const sysUser = this.sysUser ? this.sysUser : {}; + const formData = { ...sysUser, ...this.form.value }; + + delete formData.confirm; + delete formData['sex@dict']; + delete formData['status@dict']; + + if (formData.birthday) { + formData.birthday = format(formData.birthday, 'yyyy-MM-dd HH:mm:ss'); + } + + this.submit.emit(formData); + } + }); + } + + generateTitle(): string { + if (this.readonly) { + return 'Details'; + } else if (this.sysUser && this.sysUser.id) { + return 'Edit'; + } else { + return 'Add'; + } + } + + confirmValidator = (control: FormControl): { [s: string]: boolean } => { + if (!control.value) { + return { error: true, required: true }; + } else if (control.value !== this.form.controls.password.value) { + return { error: true, confirm: true }; + } + + return {}; + }; + + validateConfirmPassword = () => { + setTimeout(() => { + this.form.controls.confirm.updateValueAndValidity(); + }); + }; + + userNameValidator = (control: FormControl) => + new Observable((observer: Observer) => { + this.systemUtilsService + .duplicateCheckUsername(control.value, this.sysUser && this.sysUser.id) + .subscribe(status => { + if (status) { + observer.next(null); + } else { + observer.next({ error: true, duplicated: true }); + } + + observer.complete(); + }); + }); + + emailValidator = (control: FormControl) => + new Observable((observer: Observer) => { + if (!control.value) { + observer.next(null); + return observer.complete(); + } + + this.systemUtilsService + .duplicateCheckUserEmail(control.value, this.sysUser && this.sysUser.id) + .subscribe(status => { + if (status) { + observer.next(null); + } else { + observer.next({ error: true, duplicated: true }); + } + + observer.complete(); + }); + }); + + phoneValidator = (control: FormControl) => + new Observable((observer: Observer) => { + if (!control.value) { + observer.next(null); + return observer.complete(); + } + + this.systemUtilsService + .duplicateCheckUserPhone(control.value, this.sysUser && this.sysUser.id) + .subscribe(status => { + if (status) { + observer.next(null); + } else { + observer.next({ error: true, duplicated: true }); + } + + observer.complete(); + }); + }); +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.html b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.html new file mode 100644 index 0000000000..9ae1ea9125 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.html @@ -0,0 +1,62 @@ + + + +
+ + + Account Name + + + + + + + + Password + + + + + + + + Confirm Password + + + + + + + Please confirm your password! + + + Password is inconsistent! + + + + +
+
diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.scss b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.scss new file mode 100644 index 0000000000..510f082c6f --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.scss @@ -0,0 +1,19 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.ts new file mode 100644 index 0000000000..2478ff64be --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user-password-modal/user-password-modal.component.ts @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Component, EventEmitter, Input, OnChanges, Output, SimpleChanges } from '@angular/core'; +import { FormBuilder, FormControl, FormGroup, Validators } from '@angular/forms'; + +@Component({ + selector: 'submarine-user-password-modal', + templateUrl: './user-password-modal.component.html', + styleUrls: ['./user-password-modal.component.scss'] +}) +export class UserPasswordModalComponent implements OnChanges { + @Input() accountName: string; + @Input() visible: boolean; + @Output() readonly close: EventEmitter = new EventEmitter(); + @Output() readonly ok: EventEmitter = new EventEmitter(); + form: FormGroup; + + constructor(private fb: FormBuilder) { + this.form = this.fb.group({ + accountName: [''], + password: ['', Validators.required], + confirm: ['', this.confirmValidator] + }); + } + + ngOnChanges(changes: SimpleChanges) { + this.form.reset({ + accountName: this.accountName, + password: '', + confirm: '' + }); + } + + confirmValidator = (control: FormControl): { [s: string]: boolean } => { + if (!control.value) { + return { error: true, required: true }; + } else if (control.value !== this.form.controls.password.value) { + return { error: true, confirm: true }; + } + + return {}; + }; + + validateConfirmPassword = () => { + setTimeout(() => { + this.form.controls.confirm.updateValueAndValidity(); + }); + }; + + hideModal() { + this.close.emit(); + } + + submitForm() { + for (const key in this.form.controls) { + this.form.controls[key].markAsDirty(); + this.form.controls[key].updateValueAndValidity(); + } + + if (!this.form.valid) { + return; + } + + this.ok.emit(this.form.value.password); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.html b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.html new file mode 100644 index 0000000000..d13467abe8 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.html @@ -0,0 +1,131 @@ + + + +
+
+ + Department + + + + + + Account Name + + + + + + Email + + + + + + + + + + +
+
+ + + + + Account Name + Real Name + Department + Role + Status + Sex + Email + Create Time + Action + + + + + {{ data.userName }} + {{ data.realName }} + {{ data.deptName }} + {{ data.roleName }} + {{ data['status@dict'] }} + {{ data['sex@dict'] }} + {{ data.email }} + {{ data.createTime }} + + Edit + + More + + + +
    +
  • Details
  • +
  • Password
  • +
  • + Delete +
  • +
+
+ + + +
+
+ + + + diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/manager/user/user.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.scss similarity index 75% rename from submarine-workbench/workbench-web-ng/src/app/pages/manager/user/user.component.ts rename to submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.scss index 8c3d582c9d..937c4c810c 100644 --- a/submarine-workbench/workbench-web-ng/src/app/pages/manager/user/user.component.ts +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.scss @@ -1,4 +1,4 @@ -/* +/*! * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,15 +17,14 @@ * under the License. */ -import { Component, OnInit } from '@angular/core'; +.td-action a + a { + margin-left: 8px; +} -@Component({ - selector: 'submarine-manager-user', - templateUrl: './user.component.html', - styleUrls: ['./user.component.scss'] -}) -export class UserComponent implements OnInit { - constructor() {} +.user-table-operate { + margin-bottom: 16px; +} - ngOnInit() {} +nz-table td { + word-break: break-all; } diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.ts new file mode 100644 index 0000000000..3c544e0415 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/manager/user/user.component.ts @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Component, OnInit } from '@angular/core'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { SysUser } from '@submarine/interfaces'; +import { SysDeptSelect } from '@submarine/interfaces/sys-dept-select'; +import { DepartmentService, UserService } from '@submarine/services'; +import { NzMessageService } from 'ng-zorro-antd'; + +@Component({ + selector: 'submarine-manager-user', + templateUrl: './user.component.html', + styleUrls: ['./user.component.scss'] +}) +export class UserComponent implements OnInit { + column: string = 'createdTime'; + order: string = 'desc'; + field = [ + 'id', + 'userName', + 'realName', + 'deptName', + 'roleCode', + 'status@dict', + 'sex@dict', + 'email', + 'createTime', + 'action' + ]; + accountName: string = ''; + email: string = ''; + deptCode: string = ''; + pageNo: number = 1; + pageSize: number = 10; + userList: SysUser[] = []; + total: number = 0; + sysDeptTreeList: SysDeptSelect[] = []; + form: FormGroup; + resetPasswordModalVisible: boolean = false; + currentSysUser: SysUser; + userDrawerVisible: boolean = false; + private userDrawerReadonly: boolean = false; + + constructor( + private userService: UserService, + private deptService: DepartmentService, + private fb: FormBuilder, + private nzMessageService: NzMessageService + ) { + } + + ngOnInit() { + this.fetchUserList(); + + this.deptService.fetchSysDeptSelect().subscribe(list => { + this.sysDeptTreeList = list; + }); + + this.form = this.fb.group({ + deptCode: [this.deptCode], + accountName: [this.accountName], + email: [this.email] + }); + } + + queryUserList() { + const { deptCode, accountName, email } = this.form.getRawValue(); + this.deptCode = deptCode; + this.accountName = accountName; + this.email = email; + + this.fetchUserList(); + } + + fetchUserList() { + this.userService + .fetchUserList({ + column: this.column, + order: this.order, + field: this.field.join(','), + accountName: this.accountName, + email: this.email, + deptCode: this.deptCode ? this.deptCode : '', + pageNo: '' + this.pageNo, + pageSize: '' + this.pageSize + }) + .subscribe(({ records, total }) => { + this.total = total; + this.userList = records; + }); + } + + onShowResetPasswordModal(data: SysUser) { + this.currentSysUser = data; + this.resetPasswordModalVisible = true; + } + + onHideResetPasswordModal() { + this.currentSysUser = null; + this.resetPasswordModalVisible = false; + } + + onChangePassword(password: string) { + const { id } = this.currentSysUser; + + this.resetPasswordModalVisible = false; + + this.userService.changePassword(id, password).subscribe(() => { + this.nzMessageService.success('Change password success!'); + }, () => { + this.nzMessageService.error('Change password error'); + }); + } + + onShowUserDrawer(sysUser?: SysUser, readOnly = false) { + this.currentSysUser = sysUser; + this.userDrawerReadonly = readOnly; + this.userDrawerVisible = true; + } + + onCloseUserDrawer() { + this.userDrawerVisible = false; + } + + onSubmitUserDrawer(formData: Partial) { + this.userDrawerVisible = false; + + if (formData.id) { + this.userService.updateUser(formData).subscribe(() => { + this.nzMessageService.success('Update user success!'); + this.queryUserList(); + }, err => { + this.nzMessageService.error(err.message); + }); + } else { + this.userService.createUser(formData).subscribe(() => { + this.nzMessageService.success('Add user success!'); + this.queryUserList(); + }, err => { + this.nzMessageService.error(err.message); + }); + } + } + + onDeleteUser(data: SysUser) { + this.userService.deleteUser(data.id).subscribe( + () => { + this.nzMessageService.success('Delete user success!'); + this.fetchUserList(); + }, err => { + this.nzMessageService.success(err.message); + } + ); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench-routing.module.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench-routing.module.ts new file mode 100644 index 0000000000..3c23d98785 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench-routing.module.ts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; +import { WorkbenchComponent } from '@submarine/pages/workbench/workbench.component'; + +const routes: Routes = [ + { + path: '', + component: WorkbenchComponent, + children: [ + { + path: '', + pathMatch: 'full', + redirectTo: 'manager' + }, + { + path: 'manager', + loadChildren: () => import('./manager/manager.module').then(m => m.ManagerModule) + } + ] + } +]; + +@NgModule({ + imports: [RouterModule.forChild(routes)] +}) +export class WorkbenchRoutingModule { +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.html b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.html new file mode 100644 index 0000000000..f5b21c1fad --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.html @@ -0,0 +1,82 @@ + + + + + + + + +
+
+ +
+ +
+ + +
    +
  • + UserInfo setting +
  • +
  • +
  • + Sign out +
  • +
+
+
+
+
+ + + +
diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.scss b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.scss new file mode 100644 index 0000000000..486c21f10b --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.scss @@ -0,0 +1,107 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +:host { + display: flex; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.menu-sidebar { + position: relative; + z-index: 10; + min-height: 100vh; + box-shadow: 2px 0 6px rgba(0, 21, 41, .35); +} + +.header-trigger { + height: 64px; + font-size: 20px; + cursor: pointer; + transition: all .3s, padding 0s; +} + +.operation-user-info { + color: #333; +} + +nz-avatar { + margin-right: 8px; +} + +.trigger:hover { + color: #1890ff; +} + +.sidebar-logo { + position: relative; + height: 64px; + padding-left: 24px; + overflow: hidden; + line-height: 64px; + background: #001529; + transition: all .3s; +} + +.sidebar-logo img { + display: inline-block; + height: 32px; + width: 32px; + vertical-align: middle; +} + +.sidebar-logo h1 { + display: inline-block; + margin: 0 0 0 20px; + color: #fff; + font-weight: 600; + font-size: 20px; + font-family: Avenir, Helvetica Neue, Arial, Helvetica, sans-serif; + vertical-align: middle; +} + +nz-header { + padding: 0; + width: 100%; + z-index: 2; +} + +.app-header { + position: relative; + height: 64px; + padding: 0 24px; + background: #fff; + box-shadow: 0 1px 4px rgba(0, 21, 41, .08); + display: flex; + flex-direction: row; + justify-content: space-between; +} + +nz-content { + margin: 24px; +} + +nz-layout { + max-width: calc(100% - 256px); + + &.close { + max-width: calc(100% - 80px); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.ts new file mode 100644 index 0000000000..62732e2946 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.component.ts @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Component, OnInit } from '@angular/core'; +import { Router } from '@angular/router'; +import { UserInfo } from '@submarine/interfaces'; +import { AuthService, UserService } from '@submarine/services'; +import { NzNotificationService } from 'ng-zorro-antd'; +import { Observable } from 'rxjs'; +import { tap } from 'rxjs/operators'; + +interface SidebarMenu { + title: string; + iconType: string; + routerLink?: string; + children?: Array<{ + title: string; + routerLink?: string; + }>; +} + +@Component({ + selector: 'submarine-workbench', + templateUrl: './workbench.component.html', + styleUrls: ['./workbench.component.scss'] +}) +export class WorkbenchComponent implements OnInit { + isCollapsed: boolean = false; + menus: SidebarMenu[] = [ + { + title: 'Manager', + iconType: 'setting', + children: [ + { + title: 'User', + routerLink: '/workbench/manager/user' + }, + { + title: 'Data dict', + routerLink: '/workbench/manager/data-dict' + } + ] + } + ]; + userInfo$: Observable; + + constructor( + private router: Router, + private authService: AuthService, + private userService: UserService, + private nzNotificationService: NzNotificationService + ) { + } + + ngOnInit() { + if (this.authService.isLoggedIn) { + this.userInfo$ = this.userService.fetchUserInfo().pipe( + tap(userInfo => { + this.nzNotificationService.success('Welcome', `Welcome back, ${userInfo.name}`); + }) + ); + } + } + + logout() { + this.authService.logout().subscribe(isLogout => { + if (isLogout) { + this.router.navigate(['/user/login']); + } + }); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.module.ts b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.module.ts new file mode 100644 index 0000000000..c49facad9b --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/pages/workbench/workbench.module.ts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { CommonModule } from '@angular/common'; +import { NgModule } from '@angular/core'; +import { RouterModule } from '@angular/router'; +import { WorkbenchRoutingModule } from '@submarine/pages/workbench/workbench-routing.module'; +import { NgZorroAntdModule } from 'ng-zorro-antd'; +import { WorkbenchComponent } from './workbench.component'; + +@NgModule({ + declarations: [WorkbenchComponent], + imports: [CommonModule, WorkbenchRoutingModule, NgZorroAntdModule, RouterModule] +}) +export class WorkbenchModule { +} diff --git a/submarine-workbench/workbench-web-ng/src/app/services/auth.service.ts b/submarine-workbench/workbench-web-ng/src/app/services/auth.service.ts index 1f3865a609..099ec2fb9a 100644 --- a/submarine-workbench/workbench-web-ng/src/app/services/auth.service.ts +++ b/submarine-workbench/workbench-web-ng/src/app/services/auth.service.ts @@ -19,12 +19,12 @@ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; -import { Rest, User } from '@submarine/interfaces'; -import { BaseApiService } from '@submarine/services/base-api.service'; -import { LocalStorageService } from '@submarine/services/local-storage.service'; +import { Rest, SysUser } from '@submarine/interfaces'; import * as md5 from 'md5'; -import { Observable } from 'rxjs'; -import { map } from 'rxjs/operators'; +import { of, Observable } from 'rxjs'; +import { map, switchMap } from 'rxjs/operators'; +import { BaseApiService } from './base-api.service'; +import { LocalStorageService } from './local-storage.service'; @Injectable({ providedIn: 'root' @@ -45,26 +45,36 @@ export class AuthService { this.isLoggedIn = !!authToken; } - login(userForm: { userName: string; password: string }): Observable { - return this.httpClient - .post>(this.baseApi.getRestApi('/auth/login'), { - username: userForm.userName, - password: md5(userForm.password) - }) - .pipe( - map(res => { - if (res.success) { - this.isLoggedIn = true; - this.localStorageService.set(this.authTokenKey, res.result.token); - } + login(userForm: { userName: string; password: string }): Observable { + const apiUrl = this.baseApi.getRestApi('/auth/login'); + const params = { + username: userForm.userName, + password: md5(userForm.password) + }; - return res.success; - }) - ); + return this.httpClient.post>(apiUrl, params).pipe( + switchMap(res => { + if (res.success) { + this.isLoggedIn = true; + this.localStorageService.set(this.authTokenKey, res.result.token); + return of(res.result); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'post', params); + } + }) + ); } - logout(): void { - this.isLoggedIn = false; - this.localStorageService.remove(this.authTokenKey); + logout() { + return this.httpClient.post>(this.baseApi.getRestApi('/auth/logout'), {}).pipe( + map(res => { + if (res.result) { + this.isLoggedIn = false; + this.localStorageService.remove(this.authTokenKey); + } + + return res.result; + }) + ); } } diff --git a/submarine-workbench/workbench-web-ng/src/app/services/base-api.service.ts b/submarine-workbench/workbench-web-ng/src/app/services/base-api.service.ts index 076ab1f686..86b27108cb 100644 --- a/submarine-workbench/workbench-web-ng/src/app/services/base-api.service.ts +++ b/submarine-workbench/workbench-web-ng/src/app/services/base-api.service.ts @@ -18,6 +18,37 @@ */ import { Injectable } from '@angular/core'; +import { environment } from '@submarine/environment'; + +class HttpError extends Error { + code: number; + url: string; + method: string; + params: object; + + constructor(message: string, code: number, url?: string, method?: string, params?: object) { + super(message); + + this.code = code; + this.url = url; + this.method = method; + this.params = params; + + if (!environment.production) { + this.logError(); + } + } + + logError() { + console.group('Http error'); + console.log('error message: ', this.message); + console.log('error code: ', this.code); + console.log('-------------------------------'); + console.log('url: ', this.url); + console.log('method: ', this.method); + console.log('params: ', this.params); + } +} @Injectable({ providedIn: 'root' @@ -52,6 +83,10 @@ export class BaseApiService { return `${this.getRestApiBase()}${str}`; } + createRequestError(message: string, code: number, url?: string, method?: string, params?: any) { + return new HttpError(message, code, url, method, params); + } + private skipTrailingSlash(path) { return path.replace(/\/$/, ''); } diff --git a/submarine-workbench/workbench-web-ng/src/app/services/department.service.ts b/submarine-workbench/workbench-web-ng/src/app/services/department.service.ts new file mode 100644 index 0000000000..0872420d81 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/services/department.service.ts @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { Rest } from '@submarine/interfaces'; +import { SysDeptSelect } from '@submarine/interfaces/sys-dept-select'; +import { of, Observable } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; +import { BaseApiService } from './base-api.service'; + +@Injectable({ + providedIn: 'root' +}) +export class DepartmentService { + + constructor(private baseApi: BaseApiService, private httpClient: HttpClient) { + } + + fetchSysDeptSelect(): Observable { + const apiUrl = this.baseApi.getRestApi('/sys/dept/queryIdTree'); + return this.httpClient.get>(apiUrl).pipe( + switchMap(res => { + if (res.success) { + return of(res.result); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'get'); + } + }) + ); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/services/public-api.ts b/submarine-workbench/workbench-web-ng/src/app/services/public-api.ts index a853cff930..4bc5376921 100644 --- a/submarine-workbench/workbench-web-ng/src/app/services/public-api.ts +++ b/submarine-workbench/workbench-web-ng/src/app/services/public-api.ts @@ -18,3 +18,8 @@ */ export * from './auth.service'; +export * from './base-api.service'; +export * from './department.service'; +export * from './local-storage.service'; +export * from './system-utils.service'; +export * from './user.service'; diff --git a/submarine-workbench/workbench-web-ng/src/app/services/system-utils.service.ts b/submarine-workbench/workbench-web-ng/src/app/services/system-utils.service.ts new file mode 100644 index 0000000000..60469680f8 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/services/system-utils.service.ts @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { ListResult, Rest } from '@submarine/interfaces'; +import { SysDictItem } from '@submarine/interfaces/sys-dict-item'; +import { BaseApiService } from './base-api.service'; +import { of, Observable } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; + +export enum SysDictCode { + 'USER_SEX' = 'SYS_USER_SEX', + 'USER_STATUS' = 'SYS_USER_STATUS' +} + +@Injectable({ + providedIn: 'root' +}) +export class SystemUtilsService { + dictItemCache: { [s: string]: ListResult } = {}; + + constructor(private httpClient: HttpClient, private baseApi: BaseApiService) { + } + + fetchSysDictByCode(code: SysDictCode): Observable> { + if (this.dictItemCache[code]) { + return of(this.dictItemCache[code]); + } + + const apiUrl = `${this.baseApi.getRestApi('/sys/dictItem/getDictItems')}/${code}`; + + return this.httpClient.get>>(apiUrl).pipe( + switchMap(res => { + if (res.success) { + this.dictItemCache[code] = res.result; + return of(res.result); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'get', code); + } + }) + ); + } + + duplicateCheckUsername(userName: string, userId?: string) { + return this.duplicateCheck('sys_user', 'user_name', userName, userId); + } + + duplicateCheckUserEmail(email: string, userId?: string) { + return this.duplicateCheck('sys_user', 'email', email, userId); + } + + duplicateCheckUserPhone(phone: string, userId?: string) { + return this.duplicateCheck('sys_user', 'phone', phone, userId); + } + + private duplicateCheck( + tableName: string, + fieldName: string, + fieldVal: string, + dataId?: string + ): Observable { + const apiUrl = this.baseApi.getRestApi('/sys/duplicateCheck'); + const params = { + tableName, + fieldName, + fieldVal, + dataId: dataId + }; + + return this.httpClient.get>(apiUrl, { + params + }).pipe( + switchMap(res => { + return of(res.success); + }) + ); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/app/services/user.service.ts b/submarine-workbench/workbench-web-ng/src/app/services/user.service.ts new file mode 100644 index 0000000000..65f914d061 --- /dev/null +++ b/submarine-workbench/workbench-web-ng/src/app/services/user.service.ts @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { ListResult, Rest, SysUser, UserInfo } from '@submarine/interfaces'; +import * as md5 from 'md5'; +import { of, Observable } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; +import { BaseApiService } from './base-api.service'; + +interface UserListQueryParams { + accountName: string; + email: string; + deptCode: string; + column: string; + order: string; + field: string; + pageNo: string; + pageSize: string; +} + +@Injectable({ + providedIn: 'root' +}) +export class UserService { + private userInfo: UserInfo; + + constructor(private httpClient: HttpClient, private baseApi: BaseApiService) { + } + + fetchUserInfo(): Observable { + const apiUrl = this.baseApi.getRestApi('/sys/user/info'); + return this.httpClient.get>(apiUrl).pipe( + switchMap(res => { + if (res.success) { + this.userInfo = new UserInfo(res.result); + return of(this.userInfo); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'get'); + } + }) + ); + } + + fetchUserList(queryParams: Partial): Observable> { + const apiUrl = this.baseApi.getRestApi('/sys/user/list'); + return this.httpClient.get>>(apiUrl, { + params: queryParams + }).pipe( + switchMap(res => { + if (res.success) { + return of(res.result); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'get', queryParams); + } + }) + ); + } + + changePassword(id: string, password: string): Observable { + const apiUrl = this.baseApi.getRestApi('/sys/user/changePassword'); + + return this.httpClient.put>(apiUrl, { + id, + password: md5(password) + }).pipe( + switchMap(res => { + if (res.success) { + return of(true); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'put', { id, password }); + } + }) + ); + } + + createUser(sysUser: Partial): Observable { + const apiUrl = this.baseApi.getRestApi('/sys/user/add'); + + return this.httpClient.post>(apiUrl, sysUser).pipe( + switchMap(res => { + if (res.success) { + return of(res.result); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'post', sysUser); + } + }) + ); + } + + updateUser(sysUser: Partial): Observable { + const apiUrl = this.baseApi.getRestApi('/sys/user/edit'); + + return this.httpClient.put>(apiUrl, sysUser).pipe( + switchMap(res => { + if (res.success) { + return of(res.result); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'put', sysUser); + } + }) + ); + } + + deleteUser(id: string): Observable { + const apiUrl = this.baseApi.getRestApi(`/sys/user/delete`); + + return this.httpClient.delete>(apiUrl, { + params: { + id + } + }).pipe( + switchMap(res => { + if (res.success) { + return of(true); + } else { + throw this.baseApi.createRequestError(res.message, res.code, apiUrl, 'delete', id); + } + }) + ); + } +} diff --git a/submarine-workbench/workbench-web-ng/src/assets/logo.png b/submarine-workbench/workbench-web-ng/src/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..ff55ec2c17bbb9de800d5fd2cb9944ed0df6b3d0 GIT binary patch literal 16547 zcmeHv^K&Ln@a7xawrwXH+uq&SHa51gv2AW_+qP}n$(!W8&-Z`0tE;-YpPs3jny#6y zo}Qkrr#n(bNg4?rA07YzAj!%|r~v?A{|&(au+aa3u5-Ehe*kGAswfHo)W;!w8bJX7 zA^=$l(Vre**S>JsKZl$Bmb+V9*Vjbj?E;Jla8r$lpds+n3=hDAbfUeD6D<Ij8yi3J>HMaz z&%GU(43_y^FS6CkLiUDWS;3Caz=wxq{64fVubIv|JR2sSz2ZbjhxFwNpjBBiR+BvJBgZEpRPEqCXCf7 zh4}Nje;0eZES!9HL{C1MBE==tF^=A-hdLBUaz1t%EM1+(%}lU@`%m%YD=%|qtjq0B zjIm^X#YFlN&;&w(M*zH-9lFSeF#=D64O|Oxq^wq@LM1a*@+ahgFqg%$LQL6?>>A78 z2}@?FOf_7IB_TQm;hsg9T_kchauRSYK1#6&|Ubdl$o!%hcLiL}dAtj4lulTQN^!PmVU)`V*%e z{kSAxOc^Y+B7nrooF{Va@os!{iZ}vQ;H$HZ))_26Ni!m?I*|F-P@cB zxeJV1bufdf68*n2yX^iyE`@lx5!>ppg)D&iU@+8v^@X9_?+Ph;j4zU@zB|L|Krgz@ z8mHiWoz+}^^IuPH9W_&cg;1Z&38w0eqgN*V z+V6S^&bPTA+w`0v%LpL`fFPPcND5vGxSnb^PLR2WuOWDB%&hWpC(>@Y{%`Ho{zKXB zVMNk@cji(j$G>V%*F#6m2}(VC;&1rMmnF{cv+`XEvK>#*d^l`5Hy8f~OL#kwp!y{g ztUj#EJe5U5at=Jfglytg!EuGvtk;#B?$XT`@4=V@=m2~~l8ohQEA#p+(_sTPZ02K+ z1=;vCWGf8e<{q7%8w%3XKj<2dZR0X_HfJ$^3z>Vd2H9qv^IQGvv3mm-3l#%OjCB`( zWFMRwd(tG#vO>sEX5z(byraBCy9rpsQKy^9Rh);7vOF`Q49uQkQu#zY@AQ!R_#o0g zHf?~tI`hpZ_91Q7R(N^OZ#+s1=S{S*LV$kIoxJn!hQXic!zB#7mNXY@pvd(vj9 z7@w%4M1Gj&y*`)Wk%YwEw)1YfFMVmVpu+cJRw3jz-%}jwU!JW??3h5u%W$e^uQqMI zAgcJe(-CUc1antqV|vyQflu>PPN?hVc|_R{%4cv0`S4n@0wEBPK`+;**c z?AMr;oQ)mP^v%+3qNck6b0a6zRz-$-x9WVvQ-U&PsLk}lqTW^WWoi4D#5sB#;{WBi zT8_-3_h8vc4%8~CQz&EwKJF>K^O=4Jhv3Xfh0&z|O7)w(D%htH40T?NV>X!9F(JZi zMUW=ya-X5&s@A_-<2ak1ueiRofPX)1qcQc2^f1|DnDK*!tpvE|fh;$3;EToMhI%%m@fxx8hq{a!X7 zostg8TxcQqsx=Dd_Z=!dQ=_ENlaVxebbEbe@(zJBD0>QB*VVr8AU1^}MN9XmV%sja)z#@)L1T<*JpPgL z`oeKrd$gIwV?QBunTCg*0F?W5eJbq@@1wyW?t{tPHhiR`#No$HP40)H;1#$?b|c=n zyD0gk1dJynDn@fA5dX89qw%0nHtT*E$n9qDLeYw>kCg7vOPT`ArX-nxq&W^?Emn;X zBz8|>*3*mMyHd~GUYcC_sZhD`%uqSks`%k!5H6#66_(YI^hP$B_$9=;g!x`nrvaPg zoc?BltE5Sjp`W*Um(DYM=h8|nn*k}(%&60Kp&qsmYQGc-Q50iWSX0j_t#lXiIr)+u z6SC4u&3QEKvGJT~4a3=@4LfGx{J>1to+?A)(aMaN8xN#mMYQqN&A2^uFkf+IvFkx&J$k%U9(TQ zjsj()V#q`z5ZfNs;om!&xf`yWCc%563=sN_iA(DUL z;!8=xkt;xz+oQlc+5J&+|5xAwm9moh6r(}UbW+tD7~L|M1Q}hY$8sLhVlOXU2Vfx| zUiCEHdZQ)<;ZaB5h)1t z+v}7V!#_xh;2n}zsT#IYZ3vk+IbT=DEXf%zKrWnPlBI5>9oGP3HXvUV>JJ^N+hOw< z2%DaT$m$`~hMFK1d_G*{vxjVh$6#FynFZC^l#CWyaTe`>CY+euZzFGZD6elGS`alk z5NeH1LWr|{5YY7nzZ$Y5)UwUm&0)Endxkl#t`9XkNFww)Hc%8f)7$+yF`4Flm4K1o zq)HObzZzcr1eFY51j=7}%t(I*!t%xc7K%@6^sL@R1fcOp0zD-@yLKELG+i#;2OEm* z{snGN6bGj*l7rdydQ0Kr-yN&rb0pS3P$`S2Lx%S!w?Nu~e!*NTk5%8}2rth%c%-h$ z>7oqeNy;f{^ee7f{hfO7^%=n3)rkP_%YkSV3X~dB7X9%cm9JPVrz9m!UF_ZT*)hd@Zidz7HymL%QinwJ*Ix}UWck2Nnxt#nWpL9ECADqwQ|H} zBdOlZ$0H{3^y* zRCh9+{j_gS|9jnOt?PSvW<)~>&%-~S%{u*t(w%ebXUtKWFT^^jp*c;G=B*RPs;+h6ua7|xL-5VJVK3E6? z_PbHvxAr>hO9DwB%lSQtTV^t0w;`B>VGWQ6yo9AC{H%+q{f3SwE$Vw+^JpGfU?#Pw z5=%$r7$7`uW>2sFv&3f;&02$0)Fe{ESPMu+pPdRFO(p45q z?8~HeT$}?I-MaH{{rYLw6nE$Eq0Jetj=Tg37@dzuTR)7g&UWu*t)Bb$e3mo(*C~PA z4(v#iid^kKQv&1f@n;IQ-d3C;6f=@SyVD6T1D>P^B1LjLVWLLKNr2Kn(EueZ^p4TUF6==zv|`JmeyXBvn(iL$t1GoO)U#>Y{FXs>UQ5`dala5MK=`}7om>nH`;}Y zd<%}X!LfdH=@Wo6BuhLrZ*x+=^;s55v|gy11d7aWT7blzpTu2(Ot-?P>Q6Oy*Fm!Q z;Cl~t`JuJh=$~%$7dl9j2Cfai^%f!3m z4+X^OBi;{eQG3!Na$C?xQ^MA7R3bWX07b?C)FBJ9xKI z<|7B_alJ@0hLr85aNz>z)!pVRfK2{*mDzLktlZOrR4E;&U*ykk{kx1;>@Z7N&WN@X z3D6O{9F*iI?w-r2Xpi2a`x?-pc`%TPHbcrpU_mr7!Z|Y-wT2S4e=dl@6D&n;yrq%^ zOT1LaE4L@B4_i5;!1Jxiua0?$I-TMikCXUG_)O$Se>s3|f}kp^DZZu}SraL;oE-r= zL(!{dLg;i?ZU0b)@`~Vv$vu3$h+A(H+lr^b!jZXyu-+Zq?Pwr`B6zzmM}1~67*zWyOd4%zmKk1;+D6Msy8 z&(pdlJcQ|3ySLu^3@fv45H`wiRr&$aNiv-m-pFS(2mBgI9!3xXxb7oaPD$ravqVR{ z%SP}b^2s_}oCiO`jNRk~M-KV=(z9@X)Scz!B9}sl<&2Cd!oqw%{#+)##1)xTpb(CT!QYHnQ6ZzgE;i(4aF?Iv*_O5aoHfaZLNMi?|K{H*EoF~0++!(B$sk>)^}pE5B#_e@%oygDo@GTH;_=}>*+?zc#hBR)}45`W^gQX zlw16+&Sz!S4KR^87pIawBJ8ohit>wJCd5X_=k?Z z;TId;FhJ=))6+1~&x}ynm^&Ke-#f}DsLqTJ5tgAOfH{~gVAZ!P3|Y^My?Cjc?p5?i z{3@dU4jWk}*3f%i(kfEp>}U!?QjE)Koox2b5jk12(;9WP5;A91RI(pd0( z02sHU@{wM%$@4b*AAL!~pwx0o5I(ScK%zRfhCc!r zS1;nOPd8nee9yaGE7P}u)%GsRrQhGpyvreF8~@C&`u*@o;qQWlV;)QX&ROLlkoRFu zn6jw3;}QtTbFWi z;sn&v;)i+S^}x+guqB6mT9(|eO-b5)9+|~k=)u}`#r@eDeut+#41EzCjv=>}rxKnW zk0~P7mkNL5o=|WJq)}N>*pjR;D7tE(GQwH2FjVidP0r(^G!D@u;LxK*lJeim1&)#V(+S8=ozSI9~Q`Y#{mZJfYoqJ8jc1MBB!*=slt zqTQ_>lFH0i=!i~r`XeGOqvC`QZ-%09t+k~b(~DaRYLk{qdGoGik7V;CuR%{ zCol2&g|-+MxzFHK+iIf;s~%*pJdh-R>9}x7x{M?@Wcs!x_s^XEpf-&ARqM|8Xr0inSKeUNI5_Q z-A5+{`BY~au7Fyl@qtM8h=}U9jb_BhpzAQmVm&gwgJyDLU3)u<9axbUnSPZhtn00aN%grQ`wQ)wcBfbn@s=5}AJRt=&thDl&RF_!c~ z!_TcDK84(0^cA7*j!=n-5M(_i>Q@F0!8!HT+T96t|CvbsK%N}m%hH+z6s9X_eLS5H z-0PQycq64kCyhmh7k;_%4p?2soe4S1=*MMzMebFLg}f67pI3&ncAeo`xyGP(!-2mB zu6@;UzUP~^>;)U6voiG+Sk#k1B^*Yc-KTB+wE`75j(M0ahFHPq9A}mIn+ifi9O!>1 z^d5%_c4G&AJA~BiDQKcpMd0024hmI2J2{rIVbyI-vW#3c0!WhfK#jeI%I$u;XAK_8-fq9oC%bv?cDHF-Xcaf7=QPJ3 zbMB5$FC6LPw@z8n;d!?s8UE@l+Jfl6gBeniNif3relq|+20=N<_mu8!xruW>;XewR zL3qoDvKIhyd$pawy=g(~U@1*s zq0I&vg4x2#5^=8`$LA_?oNu zp@`Al9-^EUlxv{BNx^AU^HyFYkrMFoR3bv}&iTmwSh77bv-7aO%2v~e4Sptu%q{#> zIVp)Y1jwh=uo^VLRD~Vb`(>3qCK@ygp$i-ZgjG6eKL~0r3bs7DpSrDos`&xs5B0LC z0#uz#o;<=6D(f4vETX3_%j}nBvOs%4<#4~Ftq&53n$@@NOR@)^MZ2mHvhV7URM09v$a#f>&BAdZ_?Tq( z#EAyePX!M>D}E@vc=U8O^f*u-^vWZ#1N+1Ht60k7GLQ3eF#GgHl9fpNbX5Lr`kEVj zty%-;)1S-4qJ)ang1Rry4bE^uQaO&h_?=Lo8oq=5GO+UQLOWdl#!@#*PiEs4lz|t0o^Sh4pD+&2idHYs8t=DG41|V=h>{N*J2_r4J6d ze^usJ%N@@r>37)Pp>*zaujOjP0&`qS%fn3R?!pXUf?5ArZxGR^8dwh?o5|3xo>Nzw z--qvIC%E{2x;l(Pc5fD+=mj1a(x8m~l|R%r$~0>_oD5y4V~DP(bUl0VCe8G*{Vq7` z)$0B((<0{fX=QO+7h7Ll*2=qEvhmv#G+pszs3=o7e|V_h^i2Z8D^3AZ5CYW;EoAjV zZ1`1&$YT;0(~yn>Hp5ZqHou={q;Wp0`90&Re(c@;!u&uz14+GM`k<~r#0BMwv}drW z55|+j%YdCt8$*kb3){ZigG}z`!^JglB>e=T0l`fI6V#(qI6`m6rc&{wgUV%UXysywWuY1N}qd) z1fbbJ?Har-HByAu_EBh0QPlmr+=-*ncx32U5_d#y6*P(aL=t^vG5ot7fdOy#f@G1g z83ya=UKZXH$LDiT>s7hYhR_ z40{NU?}x#hB9%INiWK&SM5l2n#YlWY#Y}Vf2YJUKXShxSF|ut|EYh6e@~;kP3Q}Hg zLK5v&*u{K#_@Syj{U(AqBvYmk`SnuRC%n3wNQtuIT@V4bP)bL(naQfkLvh!gHvvU^ z+L26w10MhC9y5Qfvi|eh5X0=F`ztcx3Q+rCP6=~Rc_s2MRJ8P6Zp?K~vp~F8f9{KL zg9oE3&@AC0`!~0Yd!!*kj$0$1lk`Uw9UYE<4wJWI2VDDfu;=zAxF^+g0rKznPdYe% ze=O%!w)?ydNjg37Xx+MvxaYf=(W1v}?L7N0F(3pXPt)o>?zZ#i`ERNpo}NIQ(QCh# zKU5%-kZor)3_UcU z+=YGBC5@cmexVF87b*%X6p@0P&1Pq#16%3wa@q$-3Xju_*%&?hs~_AbqhpLgQTKza zYg`FuWuPO#XKN2tGEZE(lI8ilQZDRH`07*l|MkWIUG@a=aSR}WJ45?ovu zkFwk7fJ?*rBK%Te%i^Sf5Vz0ztR3<2-Mo;iq}1b=P74*(pwV-VubFK#ds>(3Gf8umqyOwVR6&q4xjMK)tq|(Cd#j@9wQe7;2)g++<+J@BlF&fL#qmL>y56 znzNXnG>A^B6`Ae76?WHKcl?@dnvyX6UaA1@7?U3p)45iU6QD&eO4imCfq+ztQ8Vne z=WsOzc2L&e^p%R|u;)y}P4?B`&T=YX6yh+nYavcQmZ>PS6pM^eO59z7bN`;i1yp}G z1q*<_i$uO$2zK~SGHG-fV@(fyX0A;#JWM{~RVuEbAzk2&Bq41*QFxF5?14X78qn$D znq_Jz7a}``mbt$tc!r0}tO0V4@uL%BN16V9v4D60>)Ae-&1~=p59sXaW*I$6uMD+PB&EWUWd1R{Nt1*J7;xt+Bbt{FfUJd92823LlYjFz zE28Yz%`j+h?+M3J^dau?Vy-`zG+Ao|wvK(UPWfOK8G;#O;@YQ!n{n2bMS>Yp#IJx+ z_Lax8x)f@I^drua5Uc&03e&#|2&@|}e+`c;WXwwWpV-(yrh_$&P%wZpKMuDa^UpiM zD89)D4+VSRB7*`kW5dzsg6(fsf*3^WREU}mP#T4y?iGyLAHg|+uV)_q^?j3wq}a&~ zM;WqfA*SbwGT>S9Rr5{ZLjIgu5OV=X3p%#2^H(Siy{4I8<>cCrBuxgW&pjO06BB)Q zyv^YNj)0f=@q^v6--4Sum>(>O#Ft?~PAdYHW8sT-vdZxSe{{+m(I6fY*HNBi`UMk{ z=ghO;7-P#z@7B36PE>4Vl!!?C5c44%8@{o#Yw0bngclj>XqVpK)A1g+S(U_`(M`i* zU`R8+=<;b=KB#T3Ya@5Dn(gPhUB22fj&z6rVf=S`7jTuDtz1x?AO=$JDSrA0^Qh;m zI2x-cDu;{~yWT#cAB&X@XiRlyX45U{t3C$9TsWh_&CFAD3eeC6aab*c+YiVNZpDNj zxa>om=ttb6kLoMsmW7f|~RAB9Vr!=1QD|*1qsPe*KhoCQ>hGz!M$P9_n7PFk?)~>*r)}OfVKU8*ME&zvS zx8J4ezs^OJZJ+oUZ9OdpffqB^9Oj0;{dC0t9cL)`cdU^2OcMWtxYVuH1g3?UC=bcc zA8>h#h$L>*iA88^5ZQtGv!ZhfWXSZT>m=cBKOpFkrB}nU{GzTt(yz~hY2}O0HlxZH z{2j$dx(TS2{Q0Cisn4e`wBsndwRJ^)pz&V*(*W|(>mI=Kp~XsJ6c?bbsvv^Vy^43k z-@yey%@J44U?}O8bbnXm6HEy6A{Vw%(B6@@q3QO<{MvUPAmE;rRT?^-AxPm@W`!8b z=~3{KKdIrxM3p2b9Y6vz0OQ*jO%F>njb-Z9Q&3uS#>XE5G9I|2iTK5!8A9TMNd^3$ zg$A_1wO1DllgmVv({>g++*^bmQ7m3wYQV<&>J)EomXz-x z3*F5fYP0W`iEe_K$q_foqM3|Az)u{A5h{ojb&Wl#((8Bhv;_vv8uAZ+I1F-DyufRaG80PgrS3#KUvyI z_Gw?Ey#4~A2f)j?HEIdizJ`}pYZ}qXZqyAGe9m+%lpnvI$Vtr>9j(R!f5g*P6e`Q0 zTer=k2H&05Kdmr-m6RoeO>OFNNPKh|I2AjEHVJ&O#gR|gP=0dGCyNTFL*urzjUa{M zpOS2~;E%3-n(DEUJjH((B<6740fzz&lf0&3-;`($c)t9Mc(||lGf6xI`E`=lw96Di zs<#?l1NUE7^C5(S*Bg5Y5T%fdL@c$4Zq^4>TQGz0$?|eWs6+C9gAqPA!Itd^sd^et zl33jCN)TpYP@v*~E2HFmF)LBEsq($}lYaBi^M4AAtdF|qazmXa{gI1=RB3lmFKLPn zC;R*ChL8U&%S=w_7Gih%bN?=?=I8A}7ca>P$cXpS4;_}oI4e+`B*26$jnq@u8tDPYMr8^>6MH}E`&t$66?;`=7CHlWuLBRdTL zN$zUhmT0BQPp8SX#I*d@Wt~CjD4?5qsy;{RyBn?(nKS7|C&r+~L+4MS7;ZVnXe09iyLJ4UM zWf&R2w(7@~Jz0ccb+%(gS*aEF)WTwFemJf!Le!XXPvOV=Soz+_w!(FgH@boi8?@n0 zoATuPu*IoyzA3maLKf@C*bh4uMQAVy)-I`Iy}Z1{!K zQeOqkJVVX+9SC#z@?B;EV+I8lI*6vO#TGA{UA~9>RIuNv6Jm$)era+z%$BlLvt%^V zW%>_CT}o?uemk_)ct_rO{Z(+J`C{&tOT{7lgj2nCJnpY{=Qn@rte3Hz@3PA8sMwQ3 zoU-ZrEQMQ~O3TF0aEcDciUbQR^2I%!YA}B3PV_{MJs$}GdPq*heoSx&ct1t*wZq@xn;RfD^r-xNk@Pk3 z%8LjySKS=<9=P-Jj{~a*C{70lskiONw-~mW&bZVisv6@yk$|eZptv4i9DeYZPRQ;r z+Of}3(or zX^=;bXkKw9J{o-ppZ_;tHluX)anuvovV~*w9p=IYG0A7B%`sn^;fP9C!aW}?+Fm6k=6!G;|8lQ8B-!##iqnek;n0f!{mls&?(m z!|gNN7K#SjhG|RG(s^xF^BX+XGJf~1q(`>T-L!wP+bVyX{rG1u+HXC03q)fyPg3bA z_A9NI@-6KR9v_s5ZC_3g9zQ9VfXMR4V#u|_q)Vg+qdg~v1|k-c2~EY~x;QHh?>s~B zL}$FZ>7<;0ID{aFk@%A9q5t`fi-)QLaD0Rzs^B>1gb+aK9`euNAU{81oH&)h*^r^pN}x720z zXo#~Rj#X5y_y$9O`$8w(kAhRWgaFJ_+Ov8!P0h+LMNG49$hn$*CUAg(_5uy@aF*K_^p1!cS}n zC17);^Tj=(4-VF<;a6XB3L*>oq^C&Bko9U6z1m^7X7i3q$XU7Mi0SNa{96FHU~@a) z8bR0+*vkWNdV?LlGII&7)6NA}`63e__+Za;WB+!35x=$6?6)#z8x&0LHIWown310} zo>ItKaeA@tyL%T;a_%4TmqpTO9c!uSHA3IGnm;JExDH{*%B_n z{4jRQgQo+n&pbrE3n?OoT}e+Nd0|_hYTJUp*Ic`t%YleAmz5+pEydYP2O&00L8qv`<B!SDW9}@)#V72^3C%zY>EZNfS z4JSfO8#L1Uv;Lbn>pb$_r+=ofRzKo`ZBm<#-4TA**6XN4c76YWnL+|_4|4n}UvHI& z1RYt#f$TQj3+-zyxAJjB;5g!TqxKOZcj9@e#aCI;6QLvKzkC$$ryQtr-c)(;JLw5O zVIn{$EfKwD&rw@K!p>P@oqY-vH8*0>ZreO^+CqyDz^!hA5fQbnu&rGypZvVaYU6H5 z{?Ip_p4J}In{Hfn<#YG^KS2+!+71rYzqMO@Y8r24EdU#zX6+=BFg^b=0FeVKKfv-i z49PU16i>t-k!>>-+~kQ9Wu$gz(pISC+gS1dL|PFI@oa}VgDXOu;hqNU#U*S-T*}`JT(F=5t$LaTe~~=E=fLZ7bjEYi+V0gz(vZZpWB2tB1H0L- z4R-fLuL6#@N+Tnyhp-@me6vaxgSxK zfW^~LSCt{b0_q$v6+IiNsdf8s@hH|i_jFGc>m+ql2~ogNg8r?UigRra=wU^AcCGh1 z+XF@tGm2Bs?7T0K&8W?H4Dl_m5~gqnPF2sCyz;DSCj{$7q3bejWoIVWuBZ3%Z7ERn z7}wgL@m4EPKy8Y6<#;|qt%5Z5^}rMQ07#}StxirxoYsa5kVQ!s*cou%^LM4DdsAj9 z7?|0KDLUyvwQ-79)}JeN@Cy7;mZ|O%IR2`jN3H&d|ILZs*fQ985tn(X@4dT4u1Q|-R$$4VP&;}nXK_P^jfVaNU}XtP*z$kbDC z-c|uV(DAjhXn~Xdf!s`fb}=v+1JMdtaKl%ZMqq9`1b#{(`giaIk{Ki6vPihe zz1n|MDxC8iucX_e1K63!YULuBFaw1hW+0YHx-EC1CIN>-@Li6@-9)l{GHZ!6u38?` zWO!Z}*yjT-8v?Dx5>ge2fiIaAMHfo6AyI}e@)j@Unuii`K8%g>A17EPgUjqOu8M%~ zak7P|!cnZiplGAnJYS2vuR$bh(U3(dn7CG;%}#PS_qj-CiR-dgTWD4&!t3?|nS#F^ zQygQP*3c{H10%~dzL9N|M ziv=Hl@D32lr$KXsn&dBj`BxGvx{Vn1zDVtPy`*|)vb}DCiM$}Lm9pzA(4D+ z_{VNJ!;{yv&!Ejm$f9XK4Bdg>)-pb-IbX8wKHv0{W@7RHB~%|>H3-j}$50EGK+BlG z*C3h#1r8wbg{#aakD`ie!a(=o5>&f9%;82c$ui@8oIx_Ka;A;?sb~l(E-Ndets3DY zuHbJ7rEA9$x(%#{#L0WOYMI0{YAQ*!oyY`(-GASFxuzs`voTlj6yKnY_d&Cr%z-%E z@=Ia2%w0H`=07=IMafgL>9{2~UGu$b`=B40D4Qn^mJqUxc_e+==0Dvf zIJM1WeKnlQFuw}^yRuPEl*}#G5!&3WC0PCIW>bpoRS3yqOJ4MjqG>!DQ_YfW1INnx zkHK8cT!f&o2|rBQ7;OZGJ{=;mxhc#y^Op{7`!+&i8tHA&%yCxJn$+vCaF{ zO*9BupJCKPz^)#_u1-BO_rY{?)~xuJf4TE)snA`|M|(@$z&7Sd5kfux6}GLG;L7yX z;)~b&%P4r^Y~PNyY48dT{s-i3q>SF;$(O~UqN7+lLXncx6F2Te%NJ*%fUu=!wQFF3 zWH@meRkeU1!|Q+KJhVF{atl@^kv_4ILi29useLg{PG5h$@oeD|^sMMafHwE1%vp*z z5SKzIluE?PR6GpVSbKz4AgOsi(8m{u98hY3>e+yo&ZK8_Wwm}{`>xcvvKRjcCY?j} zgL1ehT&M#t8u&`o3^q{_P2@nd%&+xYCw#blHjl3Dm6$l{{uio6~69<|@xnJnB+f-uEWyQIZjeh4G>)#0on~~ur{jh~-XU(K4RxQ19{i#~c zrEsH-MY7(e5&X>WH9b?R;!w*4>Yy_c!AH6O?s7-eRUBz|#3%>8sLQMz6+Yf1;N*7u z2g<=aqHRdvKTLUPD*XzgYu|P|$D_%Oa3Xw5R1MWNmBtOfn&;ZJ((_U3O?kC~Pzr^|5cOv5ms#+x`IdS z-#ae35=;SXRs950)I_A%12|sn@u(F%aX8J^LSp(0i@L^O#u7@a1EJZhFX3+G4a=iv z8GkYSNVz0(eY07!u6*rlO>toCZAOIO*m-<>wR?xgT})}Z1q1fTZh2o_&2!}1-o7h* zZ*Ss~Yn-?@E&SE}Q%|UQ_tEm5v!wO=X7fQ%qO)B@2eA)wzE&I>EA;S1@3uI*s*`98 zLEr{5%Pk2-FTTDJ@QiBPlH0xg5u6f{@}@W zX-ApVwB_n#q!>5F#mv`lQb!KSUsmSeJWyi$q5q{Oz8OqcPRXd2#JSI6x=QJI+Q#lY zq!Or!5doVnXaoEYk@YJZCyGSe6HS7bsHNV)j#vJN;e&lp=1_d#cIn~28ADja3t~eg zLeV%I}EcA|!gnv)l)IA`QEpahq~1 z`c>Z0Q2j|{{tQ$*_@Gk{!f9?Gt(HB@9bT?IaUs{=a+BcYfF||Wdn~!}U=`CUBoY$Q zx?GtL*!*I3OCsR_jeK)R`#?j5j4;_@B(*X|lxo92G zp}QCkPzUCK6kzL)DfWpCOpXV_<=Hzd8~I|wzY=lT zzK&9G)sXih?F}JVaLFL=8A^`?qGW9&E9^Kw)_pG<_x-Tj?#Qz>Z|#=3N3|7E8ftq~ z>*q#V&X9q`sU+Sp?i}nDm^3oFohsm5Ek%3^CM8mP)27Xur>2x>~|@Vj_`0?*RdcjwEN-7zQpEr44C3<~LxE z=v!Cnv4Vk{r&CoZHtnMFJG?NNJ=)tzlg@6lDYEWa=!Q0!ER$@bKmt(;-qE&M4{9*4 z$pFEx&V1`CJ(8qUktGzo2<72^v)Oe%o8rCoQCRk8F-N3dQ^@j6W7&5!9<&_8rj$&a`kq<0WBVmD}Uq0YsX*O0sJ&aMZc8=bWIg zEH(C`)Qlj-to?;W!=jG-IVLajq>fn*v6{umjU-XP83Gk>ZYDI?ISIZ4I6 z(STC8BSh4t9Nx2sx~P+#|0t?Jh*C85O1FjMYbmzm0gh}f#MFDn19-v59Tu!hp4QP| z2eu$LIK2MWMGv5xrXsCzpTCcEF)E355VN4bk9l|amyJJqU%kU>F1Ka&6J>auTTYMl z_So)}O!L5aigl5d7R?2WC{|_r4Txm<)e1fmz8cGD5VQN7FaETZ*MStnf-cwBmRWO6 zHB`_^a9It62S%#VVCI57n$n$-lnu5uK~tx02goovkR5K97PTx9sm=~#9Fb`=k9^zN zm!G$S-;0Y##Yj*0uw6ejAhyHX3F+*|V^}5liC;yZtj{-l9R=-e1#jXU&Y_ZQgytM$ zi+Q+aqo&ypZ|M=_irMa=$ZIe|Vh{HPl6P|&2HZgHR65ZwBoRnA@nZd}n9wu$^=x!*#FALG13SJ#yG}BbgDr>?u!(fR}BKN8y(P z!N`nuRHqHYi!cZHbsqEJ{yi!X24WJNv9@nwN?U@?56^Yuymias?CtU8&*i6*LZcif zfn6>yq#Lh?GYtA) Dp9g{r literal 0 HcmV?d00001 diff --git a/submarine-workbench/workbench-web-ng/src/main.ts b/submarine-workbench/workbench-web-ng/src/main.ts index 6274974adf..4613957d1b 100644 --- a/submarine-workbench/workbench-web-ng/src/main.ts +++ b/submarine-workbench/workbench-web-ng/src/main.ts @@ -27,5 +27,6 @@ if (environment.production) { enableProdMode(); } -platformBrowserDynamic().bootstrapModule(AppModule) +platformBrowserDynamic() + .bootstrapModule(AppModule) .catch(err => console.error(err)); diff --git a/submarine-workbench/workbench-web-ng/src/polyfills.ts b/submarine-workbench/workbench-web-ng/src/polyfills.ts index 15f6f9c1ae..7f618f4103 100644 --- a/submarine-workbench/workbench-web-ng/src/polyfills.ts +++ b/submarine-workbench/workbench-web-ng/src/polyfills.ts @@ -74,8 +74,7 @@ /*************************************************************************************************** * Zone JS is required by default for Angular itself. */ -import 'zone.js/dist/zone'; // Included with Angular CLI. - +import 'zone.js/dist/zone'; // Included with Angular CLI. /*************************************************************************************************** * APPLICATION IMPORTS diff --git a/submarine-workbench/workbench-web-ng/src/test.ts b/submarine-workbench/workbench-web-ng/src/test.ts index b90b1d86b4..d9617dbcf6 100644 --- a/submarine-workbench/workbench-web-ng/src/test.ts +++ b/submarine-workbench/workbench-web-ng/src/test.ts @@ -21,18 +21,12 @@ import 'zone.js/dist/zone-testing'; import { getTestBed } from '@angular/core/testing'; -import { - BrowserDynamicTestingModule, - platformBrowserDynamicTesting -} from '@angular/platform-browser-dynamic/testing'; +import { BrowserDynamicTestingModule, platformBrowserDynamicTesting } from '@angular/platform-browser-dynamic/testing'; declare const require: any; // First, initialize the Angular testing environment. -getTestBed().initTestEnvironment( - BrowserDynamicTestingModule, - platformBrowserDynamicTesting() -); +getTestBed().initTestEnvironment(BrowserDynamicTestingModule, platformBrowserDynamicTesting()); // Then we find all the tests. const context = require.context('./', true, /\.spec\.ts$/); // And load the modules. From 17e8ebc0f526f1c0e113d7a9311ce53bf333a832 Mon Sep 17 00:00:00 2001 From: Wanqiang Ji Date: Sun, 3 Nov 2019 22:58:06 +0800 Subject: [PATCH 02/23] SUBMARINE-214. [WIP] Add submitter-k8s module ### What is this PR for? Add the submitter-k8s module implements the CRUD for custom resource objects, includes the UTs. Before run the UTs, you should deploy the tf-operator in a K8s cluster and replace the config at the test resources. ### What type of PR is it? [ Feature ] ### Todos * [ ] - travics CI with UTs * [ ] - entry points * [ ] - doc ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-214 ### How should this be tested? https://travis-ci.org/jiwq/submarine/builds/606910904 ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? Yes Author: Wanqiang Ji Closes #79 from jiwq/SUBMARINE-214 and squashes the following commits: 8ed84ec [Wanqiang Ji] SUBMARINE-214. [WIP] Add submitter-k8s module --- .travis.yml | 2 +- pom.xml | 30 +++- submarine-server/server-submitter/pom.xml | 1 + .../server-submitter/submitter-k8s/pom.xml | 48 +++++++ .../server/submitter/k8s/K8sJobRequest.java | 90 ++++++++++++ .../server/submitter/k8s/K8sJobSubmitter.java | 119 +++++++++++++++ .../k8s/model/CustomResourceJob.java | 66 +++++++++ .../k8s/model/CustomResourceJobList.java | 86 +++++++++++ .../server/submitter/k8s/model/ListMeta.java | 34 +++++ .../submitter/k8s/model/ObjectMeta.java | 135 ++++++++++++++++++ .../submitter/k8s/K8SJobSubmitterTest.java | 95 ++++++++++++ .../submitter-k8s/src/test/resources/config | 19 +++ .../src/test/resources/tf_job_mnist.json | 48 +++++++ .../src/test/resources/tf_job_mnist.yaml | 29 ++++ .../resources/tfevent-volume/tfevent-pv.yaml | 15 ++ .../resources/tfevent-volume/tfevent-pvc.yaml | 14 ++ 16 files changed, 825 insertions(+), 6 deletions(-) create mode 100644 submarine-server/server-submitter/submitter-k8s/pom.xml create mode 100644 submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobRequest.java create mode 100644 submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java create mode 100644 submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/CustomResourceJob.java create mode 100644 submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/CustomResourceJobList.java create mode 100644 submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ListMeta.java create mode 100644 submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ObjectMeta.java create mode 100644 submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java create mode 100644 submarine-server/server-submitter/submitter-k8s/src/test/resources/config create mode 100644 submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_job_mnist.json create mode 100644 submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_job_mnist.yaml create mode 100644 submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pv.yaml create mode 100644 submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pvc.yaml diff --git a/.travis.yml b/.travis.yml index 1b25f562d7..037c9cb2be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ env: global: # submarine core does not required by workbench-server integration tests # If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE - - EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn" + - EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-submitter/submitter-k8s" - EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web,!submarine-workbench/workbench-server" - EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter"" - EXCLUDE_SUBMODULE_TONY="!submodules/tony,!submodules/tony/tony-mini,!submodules/tony/tony-core,!submodules/tony/tony-proxy,!submodules/tony/tony-portal,!submodules/tony/tony-azkaban,!submodules/tony/tony-cli" diff --git a/pom.xml b/pom.xml index 66e1dd15db..df70ae0e1f 100644 --- a/pom.xml +++ b/pom.xml @@ -122,6 +122,8 @@ 2.11.8 2.11 2.1.1 + + 6.0.1 @@ -137,16 +139,32 @@ + + log4j + log4j + ${log4j.version} + + + + + junit + junit + ${junit.version} + org.mockito mockito-core ${mockito.version} + + - log4j - log4j - ${log4j.version} + io.kubernetes + client-java + ${k8s.client-java.version} + + org.apache.hadoop hadoop-yarn-services-api @@ -415,6 +433,7 @@ .gitmodules LICENSE license_header + **/*.iml **/*.la **/*.svg **/*.png @@ -428,12 +447,13 @@ **/*.xsd **/*.json **/*.conf + **/*.yaml **/src/main/java/com/linkedin/tony/events/* **/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo **/src/test/resources/typicalHistFolder/job1/application123-1-1-user1-SUCCEEDED.jhist **/conf/routes - submarine-runtime/yarnservice-runtime/target/** - submarine-runtime/target/** + **/submitter-yarnservice/target/** + **/src/test/resources/config licenses/** licenses-binary/** NOTICE-binary diff --git a/submarine-server/server-submitter/pom.xml b/submarine-server/server-submitter/pom.xml index 7ab50c759c..94ffede49b 100644 --- a/submarine-server/server-submitter/pom.xml +++ b/submarine-server/server-submitter/pom.xml @@ -35,6 +35,7 @@ submitter-yarn + submitter-k8s diff --git a/submarine-server/server-submitter/submitter-k8s/pom.xml b/submarine-server/server-submitter/submitter-k8s/pom.xml new file mode 100644 index 0000000000..e33b3b43bd --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/pom.xml @@ -0,0 +1,48 @@ + + + + + + server-submitter + org.apache.submarine + 0.3.0-SNAPSHOT + + 4.0.0 + + submitter-k8s + 0.3.0-SNAPSHOT + Submarine: Kubernetes Submitter + + + + io.kubernetes + client-java + ${k8s.client-java.version} + + + + + junit + junit + + + \ No newline at end of file diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobRequest.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobRequest.java new file mode 100644 index 0000000000..827346d942 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobRequest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s; + +/** + * Job request for Kubernetes Submitter. + */ +// TODO: It should implement the JobRequest interface +public class K8sJobRequest { + private Path path; + private Object body; + private String jobName; + + public K8sJobRequest(Path path, Object body) { + this(path, body, null); + } + + public K8sJobRequest(Path path, Object body, String jobName) { + this.path = path; + this.body = body; + this.jobName = jobName; + } + + public void setPath(Path path) { + this.path = path; + } + + public Path getPath() { + return path; + } + + public void setBody(Object body) { + this.body = body; + } + + public Object getBody() { + return body; + } + + public String getJobName() { + return jobName; + } + + static class Path { + private String group; + private String apiVersion; + private String namespace; + private String plural; + + Path(String group, String apiVersion, String namespace, String plural) { + this.group = group; + this.apiVersion = apiVersion; + this.namespace = namespace; + this.plural = plural; + } + + public String getGroup() { + return group; + } + + public String getApiVersion() { + return apiVersion; + } + + public String getNamespace() { + return namespace; + } + + public String getPlural() { + return plural; + } + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java new file mode 100644 index 0000000000..18d59b6267 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s; + +import com.google.common.annotations.VisibleForTesting; +import com.google.gson.Gson; +import io.kubernetes.client.ApiClient; +import io.kubernetes.client.ApiException; +import io.kubernetes.client.Configuration; +import io.kubernetes.client.apis.CustomObjectsApi; +import io.kubernetes.client.models.V1DeleteOptions; +import io.kubernetes.client.models.V1DeleteOptionsBuilder; +import io.kubernetes.client.util.ClientBuilder; +import io.kubernetes.client.util.KubeConfig; +import org.apache.submarine.server.submitter.k8s.model.CustomResourceJob; +import org.apache.submarine.server.submitter.k8s.model.CustomResourceJobList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileReader; +import java.io.IOException; + +/** + * JobSubmitter for Kubernetes Cluster. + */ +// TODO: It should implement the JobSubmitter interface +public class K8sJobSubmitter { + private final Logger LOG = LoggerFactory.getLogger(K8sJobSubmitter.class); + + public K8sJobSubmitter(String confPath) throws IOException { + ApiClient client = + ClientBuilder.kubeconfig(KubeConfig.loadKubeConfig(new FileReader(confPath))).build(); + Configuration.setDefaultApiClient(client); + } + + public String submitJob(K8sJobRequest request) { + return "job_id"; + } + + @VisibleForTesting + CustomResourceJob createCustomJob(K8sJobRequest request) { + try { + CustomObjectsApi api = new CustomObjectsApi(); + K8sJobRequest.Path path = request.getPath(); + Object o = api.createNamespacedCustomObject(path.getGroup(), + path.getApiVersion(), path.getNamespace(), path.getPlural(), + request.getBody(), "true"); + Gson gson = new Gson(); + return gson.fromJson(gson.toJson(o), CustomResourceJob.class); + } catch (ApiException ae) { + LOG.error("Create CRD job: " + ae.getMessage(), ae); + } + return null; + } + + @VisibleForTesting + CustomResourceJob getCustomResourceJob(K8sJobRequest request) { + try { + CustomObjectsApi api = new CustomObjectsApi(); + K8sJobRequest.Path path = request.getPath(); + Object o = api.getNamespacedCustomObject(path.getGroup(), path.getApiVersion(), + path.getNamespace(), path.getPlural(), request.getJobName()); + Gson gson = new Gson(); + return gson.fromJson(gson.toJson(o), CustomResourceJob.class); + } catch (ApiException ae) { + LOG.error("Get CRD job: " + ae.getMessage(), ae); + } + return null; + } + + CustomResourceJob deleteCustomResourceJob(K8sJobRequest request) { + try { + CustomObjectsApi api = new CustomObjectsApi(); + K8sJobRequest.Path path = request.getPath(); + V1DeleteOptions body = + new V1DeleteOptionsBuilder().withApiVersion(path.getApiVersion()).build(); + Object o = api.deleteNamespacedCustomObject(path.getGroup(), + path.getApiVersion(), path.getNamespace(), path.getPlural(), + request.getJobName(), body, null, null, null); + Gson gson = new Gson(); + return gson.fromJson(gson.toJson(o), CustomResourceJob.class); + } catch (ApiException ae) { + LOG.error("Delete CRD job: " + ae.getMessage(), ae); + } + return null; + } + + @VisibleForTesting + CustomResourceJobList listCustomResourceJobs(K8sJobRequest request) { + try { + CustomObjectsApi api = new CustomObjectsApi(); + K8sJobRequest.Path path = request.getPath(); + Object o = api.listNamespacedCustomObject(path.getGroup(), path.getApiVersion(), + path.getNamespace(), path.getPlural(), "true", null, null, null, null, null); + Gson gson = new Gson(); + return gson.fromJson(gson.toJson(o), CustomResourceJobList.class); + } catch (ApiException ae) { + LOG.error("List CRD jobs: " + ae.getMessage(), ae); + } + return null; + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/CustomResourceJob.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/CustomResourceJob.java new file mode 100644 index 0000000000..610fc79984 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/CustomResourceJob.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s.model; + +import com.google.gson.GsonBuilder; +import com.google.gson.annotations.SerializedName; + +import java.util.Map; + +/** + * The response job for CRD API. + * POST: /apis/{group}/{version}/namespaces/{namespace}/{plural} + * GET: /apis/{group}/{version}/namespaces/{namespace}/{plural}/{name} + * DELETE: /apis/{group}/{version}/namespaces/{namespace}/{plural}/{name} + */ +public class CustomResourceJob { + @SerializedName("apiVersion") + private String apiVersion; + + @SerializedName("kind") + protected String kind; + + @SerializedName("metadata") + private ObjectMeta metadata; + + @SerializedName("spec") + private Map spec; + + public String getApiVersion() { + return apiVersion; + } + + public String getKind() { + return kind; + } + + public ObjectMeta getMetadata() { + return metadata; + } + + public Map getSpec() { + return spec; + } + + @Override + public String toString() { + return new GsonBuilder().setPrettyPrinting().create().toJson(this); + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/CustomResourceJobList.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/CustomResourceJobList.java new file mode 100644 index 0000000000..3ad0114baa --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/CustomResourceJobList.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s.model; + +import com.google.gson.GsonBuilder; +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * The response for CRD API. + * GET: /apis/{group}/{version}/namespaces/{namespace}/{plural} + */ +public class CustomResourceJobList { + @SerializedName("apiVersion") + private String apiVersion; + + @SerializedName("items") + private List items = new ArrayList<>(); + + @SerializedName("kind") + protected String kind; + + @SerializedName("metadata") + private ListMeta metadata; + + public String getApiVersion() { + return apiVersion; + } + + public List getItems() { + return items; + } + + public String getKind() { + return kind; + } + + public ListMeta getMetadata() { + return metadata; + } + + @Override + public int hashCode() { + return Objects.hash(apiVersion, items, kind, metadata); + } + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CustomResourceJobList jobList = (CustomResourceJobList) o; + return Objects.equals(this.apiVersion, jobList.apiVersion) && + Objects.equals(this.items, jobList.items) && + Objects.equals(this.kind, jobList.kind) && + Objects.equals(this.metadata, jobList.metadata); + } + + @Override + public String toString() { + return new GsonBuilder().setPrettyPrinting().create().toJson(this); + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ListMeta.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ListMeta.java new file mode 100644 index 0000000000..7ed10f51f0 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ListMeta.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s.model; + +import com.google.gson.GsonBuilder; +import io.kubernetes.client.models.V1ListMeta; + +/** + * ListMeta describes metadata that synthetic resources must have, including lists and various + * status objects. + */ +public class ListMeta extends V1ListMeta { + @Override + public String toString() { + return new GsonBuilder().setPrettyPrinting().create().toJson(this); + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ObjectMeta.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ObjectMeta.java new file mode 100644 index 0000000000..95244fd45a --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/ObjectMeta.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s.model; + +import com.google.gson.GsonBuilder; +import com.google.gson.annotations.SerializedName; + +import java.util.List; +import java.util.Map; + +/** + * ObjectMeta is metadata that all persisted resources must have, which includes all objects users + * must create. + */ +public class ObjectMeta { + @SerializedName("annotations") + private Map annotations; + + @SerializedName("clusterName") + private String clusterName; + + @SerializedName("creationTimestamp") + private String creationTimestamp; + + @SerializedName("deletionGracePeriodSeconds") + private Long deletionGracePeriodSeconds; + + @SerializedName("deletionTimestamp") + private String deletionTimestamp; + + @SerializedName("finalizers") + private List finalizers; + + @SerializedName("generateName") + private String generateName; + + @SerializedName("generation") + private Long generation; + + @SerializedName("labels") + private Map labels = null; + + @SerializedName("name") + private String name = null; + + @SerializedName("namespace") + private String namespace = null; + + @SerializedName("resourceVersion") + private String resourceVersion = null; + + @SerializedName("selfLink") + private String selfLink = null; + + @SerializedName("uid") + private String uid = null; + + public Map getAnnotations() { + return annotations; + } + + public String getClusterName() { + return clusterName; + } + + public String getCreationTimestamp() { + return creationTimestamp; + } + + public Long getDeletionGracePeriodSeconds() { + return deletionGracePeriodSeconds; + } + + public String getDeletionTimestamp() { + return deletionTimestamp; + } + + public List getFinalizers() { + return finalizers; + } + + public String getGenerateName() { + return generateName; + } + + public Long getGeneration() { + return generation; + } + + public Map getLabels() { + return labels; + } + + public String getName() { + return name; + } + + public String getNamespace() { + return namespace; + } + + public String getResourceVersion() { + return resourceVersion; + } + + public String getSelfLink() { + return selfLink; + } + + public String getUid() { + return uid; + } + + @Override + public String toString() { + return new GsonBuilder().setPrettyPrinting().create().toJson(this); + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java new file mode 100644 index 0000000000..70b9c00285 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.k8s; + +import org.apache.submarine.server.submitter.k8s.model.CustomResourceJob; +import org.apache.submarine.server.submitter.k8s.model.CustomResourceJobList; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; + +public class K8SJobSubmitterTest { + private final String jobName = "mnist"; + private K8sJobSubmitter submitter; + private K8sJobRequest.Path path; + + @Before + public void before() throws IOException { + String confPath = this.getClass().getResource("/config").getFile(); + submitter = new K8sJobSubmitter(confPath); + path = new K8sJobRequest.Path("kubeflow.org","v1","kubeflow", "tfjobs"); + } + + @Test + public void testCreateCustomJob() throws URISyntaxException { + if (getCustomJob() != null) { + K8sJobRequest request = new K8sJobRequest(path, null, jobName); + CustomResourceJob delJob = submitter.deleteCustomResourceJob(request); +// Assert.assertNotNull(delJob); + } + + CustomResourceJob job = submitter.createCustomJob(new K8sJobRequest(path, getCutomJobSpecFile())); +// Assert.assertNotNull(job); + System.out.println("Create job: " + job); + } + + @Test + public void testGetCustomJob() throws URISyntaxException { + testCreateCustomJob(); + + CustomResourceJob job = getCustomJob(); +// Assert.assertNotNull(job); +// Assert.assertEquals(job.getMetadata().getName(), jobName); + System.out.println("Get Job: " + job); + } + + @Test + public void testListCustomJobs() throws URISyntaxException { + CustomResourceJobList list = submitter.listCustomResourceJobs(new K8sJobRequest(path, getCutomJobSpecFile())); + System.out.println("Job List: " + list); + } + + @Test + public void testDeleteCustomJob() throws URISyntaxException { + if (getCustomJob() == null) { + CustomResourceJob job = submitter.createCustomJob(new K8sJobRequest(path, getCutomJobSpecFile())); +// Assert.assertNotNull(job); + } + + K8sJobRequest request = new K8sJobRequest(path, null, jobName); + CustomResourceJob delJob = submitter.deleteCustomResourceJob(request); +// Assert.assertNotNull(delJob); + System.out.println("Delete job: " + delJob); + } + + private CustomResourceJob getCustomJob() { + K8sJobRequest request = new K8sJobRequest(path, null, jobName); + return submitter.getCustomResourceJob(request); + } + + private File getCutomJobSpecFile() throws URISyntaxException { + URL fileUrl = this.getClass().getResource("/tf_job_mnist.json"); + return new File(fileUrl.toURI()); + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/resources/config b/submarine-server/server-submitter/submitter-k8s/src/test/resources/config new file mode 100644 index 0000000000..bc57012719 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/test/resources/config @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1Ea3hNREl6TVRjeE5sb1hEVEk1TURrd09ESXpNVGN4Tmxvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTm8zCmJGbnllQmlhOUlzeHorVWg3cExGMWNaTkNiR0lkWXJxejFFUlRodXcwN2t1QnlDNWp4K2wxOEZCcGkvSVJ0OWYKaXU4MVVGZEVKSStKaEo2bC95Y3RtRitxaXhiMkUyM1ltL29VTGIxOVh1V3RuZXZHZkNNZFZiUVRvWEl5MC9xVwpLdlQrQ0J0dFVqMVRoUnI3MFVaeXU2NFBoVC96S1BKajh3UTNjVVozNDdYb2pMOElMZTZ1a043UG03NmJ5b2FUCklYUGpmWWcvbEM0dHdKRVJiL0prSEIrZFczWWcvTHhRdXZQL2ZIbjZoZTJaS0Q5Y1dLamhYaU01U2RIY0ZYRFIKanJBRk5QaDB5a2JRNkhac2lKdTdPZmhzSGlTWXR6UStKY3JjdUxCVHpJV3RBNUNEZG1XcEFLWUVxVWQyZEczVgpOeE1XSEQyTkRFZkgxYkg1RWNzQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUNCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDQ25MQUQxVGFWWElKdmRUU3MvQ1pCZnk3Z2VpbFA5V2xGRW5jQ0duNlFPeHlUcXA0SQpKNElVT05PUkhmdmlwVmtaMDVFNy91RjJjRU1YRFd2djJrbGp4cE51WmQwYmVzUklac3ZSZzQ1OTgvbVVtQzZJCk1XYmdubC90QWdIMzJDNzhGeUpnNUpmU2F0V0xBVWFUcDNDTjlPMWY5Qy90NktnSE5KMUVzVFNsRG1IcEV1bXgKdFVUbkN3aUpXalNwQjFKYUdnNGZjUXU3MWtLcXB2MWlTSjdmY3p4dUw0U0hGL0dxTUVqdXRoYXhxOCtGdTBSVQoxVkc0ODBYbmVTWjJyd3l4bytHcVhZMXBTMTVDQ1RmV0JMNEVaMUp1eVhaRy90b3ZJdmVZL0pVbytKLzl6SS9TCmdwNU9JcWlENDZxckVZMjJENHRadENrTTZWOGRrWEQ5TysxZgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + server: https://localhost:8443 + name: kind +contexts: +- context: + cluster: kind + user: kind-admin + name: kind +current-context: kind +kind: Config +preferences: {} +users: +- name: kind-admin + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJVzVPRkVhcmhlSU13RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2JXbHVhV3QxWW1WRFFUQWVGdzB4T1RBNU1UQXlNekUzTVRaYUZ3MHlNREE1TVRBeU16RTNORFJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXUyRWJyaVMraDZ4MzdaeXYKMnpvZWlUalI5Zm4rYSt3ZTJ4TXFHaDJVMG1vRGV6Y251TkdXWUZXUWpob0VrOEhnQTJGMmFZbEhJeEMwb00zaQp6OUdCcWFyNGNRY0VXanp2UjlaRzNpcGdYODU5NFhCSFJKOG9WUnQ5dWVuYU9NT04wZVFWWlhYcnBNTmV2NjArCkh5Yk9uTk1BYWxSdEY3WnF4eEVuVTdEZXBlN2ZJcnVkVmQxbzVVN2Z3bFBLdVpiWURFaGM3Qi9qbjI1bVdQSTUKRE8rWTFBM1NERXBCSXFlVVpRUWNlQm9udU1lMXRqajRMTXlIWWROaTRWcm9TaGZRaTFZbkEvVjNTcFdqTWorRgpBRitoOFo1aDczOVQ5c1QxbEJuMnYwa2ZBSmpLdGNkQWRubXA3UXh5SDVueHh4UFBlMDQxV1RQd1JyeVlNOElTCmVNNGx1UUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFFWlhyMEd4WEVlTk1URVlGUFh3SlVTL24vekZDZVVXZ3RZdQorRGZ5SzJDN0dNb05qSzl5NDBkUVYvMmNvSzF5Z1RwZ1J6QTFoSUZ4aGVST054dmwxMlV0a1F3ckRjanZ3TDdhCkU0aVdXOWltcXdzR0pwcmpOSTNUOTJ0eVNMcTJodFArVkFTVUtxYkdJRWtSSTJ4NVdoQ0ZwUXdsZk5RbFJNU1MKVmlqbW16S21DelF6QmpNUVE0cTdUZzRkV3g0a1B1NEU4eHNOK2lGR3A0WFRwK0gxZEtRSHBWT2ZYaklza2hLSgoybEdCd1Q2QXpjL0tvOEdjcDZCWElyd1hpYit6eHlzSVZkeGE2akF5dm5aMUc5eGUwTVJuK0s0eWQzRWdRdEpWCnRxNWVTYVI1R2lIaEI0Rm5zYXM0RVFpdEY3dXpqWk1haEc3U3lYT1pVSWw2TURYdmlGVT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBdTJFYnJpUytoNngzN1p5djJ6b2VpVGpSOWZuK2Erd2UyeE1xR2gyVTBtb0RlemNuCnVOR1dZRldRamhvRWs4SGdBMkYyYVlsSEl4QzBvTTNpejlHQnFhcjRjUWNFV2p6dlI5WkczaXBnWDg1OTRYQkgKUko4b1ZSdDl1ZW5hT01PTjBlUVZaWFhycE1OZXY2MCtIeWJPbk5NQWFsUnRGN1pxeHhFblU3RGVwZTdmSXJ1ZApWZDFvNVU3ZndsUEt1WmJZREVoYzdCL2puMjVtV1BJNURPK1kxQTNTREVwQklxZVVaUVFjZUJvbnVNZTF0amo0CkxNeUhZZE5pNFZyb1NoZlFpMVluQS9WM1NwV2pNaitGQUYraDhaNWg3MzlUOXNUMWxCbjJ2MGtmQUpqS3RjZEEKZG5tcDdReHlINW54eHhQUGUwNDFXVFB3UnJ5WU04SVNlTTRsdVFJREFRQUJBb0lCQVFDbmJsNE5mVWZDYzVtMQorQXdyR0FPeEdkU0N0cDlnVDl4Q1pSMTV1SFVDanpnTklYdWM2YW8wNmtBQXViN2RTSitpWmJRcEdVRTNVL1lTCnVlV3hUTU5QbURYbXlSNWFnNWhMMkd2Nm1HTUZPTDNDYzFFN1Avd0tFRWFIbVM0bENwZnV0SjlweWJTRnhRMGsKdU9pSmJXT3hQZGxoS1k5UllPNkljZ3R4L05qMTZyUEFabHJqMUdzcmpQSklNUVR0aVNnNkk2c2Q0Vlp6bWlEMQpPNnJYT1g4ODFVVEg0QUpHVjR4VEkwcTdYUEpINnRGek5nQU8wV2c2Z0U5dnU3L0E4ay81Zit3YmdHSjg4RUgzCllYQlBKRGV3bDBONW1sZE1Da1hJL0w5WmZxZkxqTm9wV2JPMzdZeTJBeXFEU2pLcGF1bjRSWnhwZ2tMUk5VM0IKOHJPUHJPY0JBb0dCQU0xcEhnRXdJNzlTemN1RnBtbHFkRzZPck5NQmVycE1qMkhDZUFnQlFEUDhqS09Ud05DYQpqWmhTak9pd3BRbXZYSnBjU3dobVhIakFlNUdMY2crSWYyYjJsZVozTFJLcVlERnNOSTNEQWZNOFpTS3dMbXlaCmI3RjVVY3VWU3pYWEpzcEJ1UFE4S3BMQ01JZnU3SVhvVWpMMlRXbkt4dEUxanhWV1JmcnBzd1ZIQW9HQkFPbUgKSkJERmphdlBoL0Z6bzg5bzBBK3FtYko2WmZ0WkdBRGlMRVdxdU03cjFzbWZSQUpvN3RFVU4wZ1J0TFU4eFYySApPTEdrZ2dDVzBGd3RTdGVtekJrWG5vT1QzTk51VTJMRm03Z1RscnBlSlc5THhwNHNnUjBoZkUrMzJZaVh1cFNBCkk3YS9WdWR5K1NaUW56SktFQ1V0eUhmNXVtMVNIeHRpWVAvRVp2ei9Bb0dBRlhBN0tYU1V4b3NsMzIvN2hsVU8KQWR2eVBiT1IwNC9hQnlIODFSa01VbUN4YlFFbXhwN2EyR01GUWVQdDFhenZ4VXlNcUR2R3V2T2I4ZnpUMHJzMApVTks3b21vZDBpem1EVUoxMmJYWXBrSlBhK2RuY1A0eW9ocy9aamdXdVcvVEpKam5iTlk2dTBaNU9qYTlpTnhPCmQvTXM4VFd3dW5VVWtlMkhNRURqNWxFQ2dZRUFsNDdLWEV3cUhyaEliTzUxQUxYd1drcUpCMmtpYUphdzJsbnYKQzZPNTN5ajJNOUkyVEM0WDEvOTd4VDBnZjRNYVFCSHpQRzhjcEE4ODJLWFliYzdscE45TVUrNlJvWEoza2FIWAo1d1puaXpwd3B5T1JtclFkYzI5NUF5KzVjVW12b0pMdEdyeGhSVUs3Nmg3bXdyZUlkT2lhc3lxUzBFSTh5QVA0CkdlWTRSTGNDZ1lFQWlpUDI5blBYNzI4WmF5WWhkT1ZYbFpwZkwyRm9hbmxlWUhEVUJvaHd5U01NSE5heWJCZFcKQVE5T3laVHNuZERKQjF2cFVyMHRkMVltb3N0YURyTXdwU1dsZlhRSWgxY25iSjMvWm5WdTNFa2NQeGxCM1BBWApIQ2N4WDZSemNBeVMxQUhwWDlwM3hFUzl6UEYyWHVRelByc0VhRExOUlFBNFA4Ris1YkU4TDdNPQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_job_mnist.json b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_job_mnist.json new file mode 100644 index 0000000000..78be457080 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_job_mnist.json @@ -0,0 +1,48 @@ +{ + "apiVersion": "kubeflow.org/v1", + "kind": "TFJob", + "metadata": { + "name": "mnist", + "namespace": "kubeflow" + }, + "spec": { + "cleanPodPolicy": "None", + "tfReplicaSpecs": { + "Worker": { + "replicas": 1, + "restartPolicy": "Never", + "template": { + "spec": { + "containers": [ + { + "name": "tensorflow", + "image": "gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0", + "command": [ + "python", + "/var/tf_mnist/mnist_with_summaries.py", + "--log_dir=/train/logs", + "--learning_rate=0.01", + "--batch_size=150" + ], + "volumeMounts": [ + { + "mountPath": "/train", + "name": "training" + } + ] + } + ], + "volumes": [ + { + "name": "training", + "persistentVolumeClaim": { + "claimName": "tfevent-volume" + } + } + ] + } + } + } + } + } +} diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_job_mnist.yaml b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_job_mnist.yaml new file mode 100644 index 0000000000..370f4cc766 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tf_job_mnist.yaml @@ -0,0 +1,29 @@ +apiVersion: "kubeflow.org/v1" +kind: "TFJob" +metadata: + name: "mnist" + namespace: kubeflow +spec: + cleanPodPolicy: None + tfReplicaSpecs: + Worker: + replicas: 1 + restartPolicy: Never + template: + spec: + containers: + - name: tensorflow + image: gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0 + command: + - "python" + - "/var/tf_mnist/mnist_with_summaries.py" + - "--log_dir=/train/logs" + - "--learning_rate=0.01" + - "--batch_size=150" + volumeMounts: + - mountPath: "/train" + name: "training" + volumes: + - name: "training" + persistentVolumeClaim: + claimName: "tfevent-volume" diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pv.yaml b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pv.yaml new file mode 100644 index 0000000000..a450c6a492 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pv.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: tfevent-volume + labels: + type: local + app: tfjob +spec: + capacity: + storage: 10Gi + storageClassName: standard + accessModes: + - ReadWriteMany + hostPath: + path: /tmp/data diff --git a/submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pvc.yaml b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pvc.yaml new file mode 100644 index 0000000000..7d7f8487a1 --- /dev/null +++ b/submarine-server/server-submitter/submitter-k8s/src/test/resources/tfevent-volume/tfevent-pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: tfevent-volume + namespace: kubeflow + labels: + type: local + app: tfjob +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi From d2045ce9e484c51da270f57b1bf7fc98c187c522 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Fri, 25 Oct 2019 15:12:32 +0200 Subject: [PATCH 03/23] SUBMARINE-67. Add tests to Localizer class ### What is this PR for? The goal is to add tests for the Localizer class. Also the class itself has been refactored a bit (methods from `handleLocalizations` have been decoupled). ### What type of PR is it? Refactoring | Test ### Todos No TODOs. ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-67 ### How should this be tested? * No integration tests is needed. * The new unit tests should pass. ### Screenshots (if appropriate) Not needed. ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Adam Antal Closes #64 from adamantal/SUBMARINE-67 and squashes the following commits: f07b599 [Adam Antal] SUBMARINE-67. Add tests to Localizer class --- .../yarnservice/utils/Localizer.java | 222 ++++++++---- .../yarnservice/utils/LocalizerTest.java | 340 ++++++++++++++++++ 2 files changed, 484 insertions(+), 78 deletions(-) create mode 100644 submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/LocalizerTest.java diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/utils/Localizer.java b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/utils/Localizer.java index 34cc0c7054..1fcdc2a549 100644 --- a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/utils/Localizer.java +++ b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/utils/Localizer.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.api.records.ConfigFile.TypeEnum; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.submarine.client.cli.param.Localization; import org.apache.submarine.client.cli.param.runjob.RunJobParameters; @@ -63,7 +64,7 @@ public Localizer(FileSystemOperations fsOperations, * If remoteUri is directory, we'll download it, zip it and upload * to HDFS. * If localFilePath is ".", we'll use remoteUri's file/dir name - * */ + */ public void handleLocalizations(Service service) throws IOException { // Handle localizations @@ -71,22 +72,113 @@ public void handleLocalizations(Service service) remoteDirectoryManager.getJobStagingArea( parameters.getName(), true); List localizations = parameters.getLocalizations(); - String remoteUri; - String containerLocalPath; // Check to fail fast - for (Localization loc : localizations) { - remoteUri = loc.getRemoteUri(); + checkFilesExist(localizations); + + // Start download remote if needed and upload to HDFS + for (Localization localization : localizations) { + LocalizationState localizationState = new LocalizationState(localization, + remoteDirectoryManager); + Path resourceToLocalize = new Path(localizationState.remoteUri); + String sourceFile = determineSourceFile(localizationState); + + if (localizationState.needUploadToHDFS) { + resourceToLocalize = + fsOperations.uploadToRemoteFile(stagingDir, sourceFile); + } + if (localizationState.needToDeleteTempFile) { + fsOperations.deleteFiles(sourceFile); + } + // Remove .zip from zipped dir name + if (isZippedArchive(sourceFile, localizationState.destFileType)) { + // Delete local zip file + fsOperations.deleteFiles(sourceFile); + sourceFile = getNameUntilUnderscore(sourceFile); + } + + String containerLocalPath = localizationState.containerLocalPath; + // If provided, use the name of local uri + if (!containerLocalPath.equals(".") + && !containerLocalPath.equals("./")) { + // Change the YARN localized file name to what will be used in container + sourceFile = getLastNameFromPath(containerLocalPath); + } + String localizedName = getLastNameFromPath(sourceFile); + LOG.info("The file or directory to be localized is {}. " + + "Its localized filename will be {}", + resourceToLocalize.toString(), localizedName); + ConfigFile configFile = new ConfigFile() + .srcFile(resourceToLocalize.toUri().toString()) + .destFile(localizedName) + .type(localizationState.destFileType); + service.getConfiguration().getFiles().add(configFile); + + if (containerLocalPath.startsWith("/")) { + addToMounts(service, localization, containerLocalPath, sourceFile); + } + } + } + + private String determineSourceFile(LocalizationState localizationState) throws IOException { + if (localizationState.directory) { + // Special handling of remoteUri directory. + return fsOperations.downloadAndZip(localizationState.remoteUri, + getLastNameFromPath(localizationState.remoteUri), true); + } else if (localizationState.remote && + !needHdfs(localizationState.remoteUri)) { + // Non HDFS remote URI. + // Non directory, we don't need to zip + return fsOperations.downloadAndZip(localizationState.remoteUri, + getLastNameFromPath(localizationState.remoteUri), false); + } + return localizationState.remoteUri; + } + + private static String getNameUntilUnderscore(String sourceFile) { + int suffixIndex = sourceFile.lastIndexOf('_'); + if (suffixIndex == -1) { + throw new IllegalStateException(String.format( + "Vale of archive filename" + + " supposed to contain an underscore. Filename was: '%s'", + sourceFile)); + } + sourceFile = sourceFile.substring(0, suffixIndex); + return sourceFile; + } + + private static boolean isZippedArchive(String sourceFile, + TypeEnum destFileType) { + return destFileType == TypeEnum.ARCHIVE + && sourceFile.endsWith(".zip"); + } + + // set mounts + // if mount path is absolute, just use it. + // if relative, no need to mount explicitly + private static void addToMounts(Service service, Localization loc, + String containerLocalPath, String sourceFile) { + String mountStr = getLastNameFromPath(sourceFile) + ":" + + containerLocalPath + ":" + loc.getMountPermission(); + LOG.info("Add bind-mount string {}", mountStr); + appendToEnv(service, + EnvironmentUtilities.ENV_DOCKER_MOUNTS_FOR_CONTAINER_RUNTIME, + mountStr, ","); + } + + private void checkFilesExist(List localizations) + throws IOException { + String remoteUri; + for (Localization localization : localizations) { + remoteUri = localization.getRemoteUri(); Path resourceToLocalize = new Path(remoteUri); - // Check if remoteUri exists + if (remoteDirectoryManager.isRemote(remoteUri)) { - // check if exists if (!remoteDirectoryManager.existsRemoteFile(resourceToLocalize)) { throw new FileNotFoundException( "File " + remoteUri + " doesn't exists."); } } else { - // Check if exists File localFile = new File(remoteUri); if (!localFile.exists()) { throw new FileNotFoundException( @@ -96,78 +188,52 @@ public void handleLocalizations(Service service) // check remote file size fsOperations.validFileSize(remoteUri); } - // Start download remote if needed and upload to HDFS - for (Localization loc : localizations) { - remoteUri = loc.getRemoteUri(); - containerLocalPath = loc.getLocalPath(); - String srcFileStr = remoteUri; - ConfigFile.TypeEnum destFileType = ConfigFile.TypeEnum.STATIC; - Path resourceToLocalize = new Path(remoteUri); - boolean needUploadToHDFS = true; - - - // Special handling of remoteUri directory - boolean needDeleteTempFile = false; - if (remoteDirectoryManager.isDir(remoteUri)) { - destFileType = ConfigFile.TypeEnum.ARCHIVE; - srcFileStr = fsOperations.downloadAndZip( - remoteUri, getLastNameFromPath(srcFileStr), true); - } else if (remoteDirectoryManager.isRemote(remoteUri)) { - if (!needHdfs(remoteUri)) { - // Non HDFS remote uri. Non directory, no need to zip - srcFileStr = fsOperations.downloadAndZip( - remoteUri, getLastNameFromPath(srcFileStr), false); - needDeleteTempFile = true; - } else { - // HDFS file, no need to upload - needUploadToHDFS = false; - } - } + } - // Upload file to HDFS - if (needUploadToHDFS) { - resourceToLocalize = - fsOperations.uploadToRemoteFile(stagingDir, srcFileStr); - } - if (needDeleteTempFile) { - fsOperations.deleteFiles(srcFileStr); - } - // Remove .zip from zipped dir name - if (destFileType == ConfigFile.TypeEnum.ARCHIVE - && srcFileStr.endsWith(".zip")) { - // Delete local zip file - fsOperations.deleteFiles(srcFileStr); - int suffixIndex = srcFileStr.lastIndexOf('_'); - srcFileStr = srcFileStr.substring(0, suffixIndex); - } - // If provided, use the name of local uri - if (!containerLocalPath.equals(".") - && !containerLocalPath.equals("./")) { - // Change the YARN localized file name to what'll used in container - srcFileStr = getLastNameFromPath(containerLocalPath); - } - String localizedName = getLastNameFromPath(srcFileStr); - LOG.info("The file/dir to be localized is {}", - resourceToLocalize.toString()); - LOG.info("Its localized file name will be {}", localizedName); - service.getConfiguration().getFiles().add(new ConfigFile().srcFile( - resourceToLocalize.toUri().toString()).destFile(localizedName) - .type(destFileType)); - // set mounts - // if mount path is absolute, just use it. - // if relative, no need to mount explicitly - if (containerLocalPath.startsWith("/")) { - String mountStr = getLastNameFromPath(srcFileStr) + ":" - + containerLocalPath + ":" + loc.getMountPermission(); - LOG.info("Add bind-mount string {}", mountStr); - appendToEnv(service, - EnvironmentUtilities.ENV_DOCKER_MOUNTS_FOR_CONTAINER_RUNTIME, - mountStr, ","); - } - } + private enum LocalizationType { + REMOTE_FILE, REMOTE_DIRECTORY, LOCAL_FILE, LOCAL_DIRECTORY + } + + private static String getLastNameFromPath(String sourceFile) { + return new Path(sourceFile).getName(); } - private String getLastNameFromPath(String srcFileStr) { - return new Path(srcFileStr).getName(); + private static class LocalizationState { + private final String remoteUri; + private final LocalizationType localizationType; + private final boolean needHdfs; + private final boolean needUploadToHDFS; + private final boolean needToDeleteTempFile; + private final String containerLocalPath; + private final TypeEnum destFileType; + private final boolean directory; + private final boolean remote; + + LocalizationState(Localization localization, + RemoteDirectoryManager remoteDirectoryManager) throws IOException { + this.remoteUri = localization.getRemoteUri(); + this.directory = remoteDirectoryManager.isDir(remoteUri); + this.remote = remoteDirectoryManager.isRemote(remoteUri); + this.localizationType = determineLocalizationType(directory, remote); + this.needHdfs = determineNeedHdfs(remote); + //HDFS file don't need to be uploaded + this.needUploadToHDFS = + directory || (remote && !this.needHdfs) || !remote; + this.needToDeleteTempFile = remote && !this.needHdfs; + this.containerLocalPath = localization.getLocalPath(); + this.destFileType = directory ? TypeEnum.ARCHIVE : TypeEnum.STATIC; + } + + private boolean determineNeedHdfs(boolean remote) { + return remote && needHdfs(remoteUri); + } + + private LocalizationType determineLocalizationType(boolean directory, boolean remote) { + if (directory) { + return remote ? LocalizationType.REMOTE_DIRECTORY : LocalizationType.LOCAL_DIRECTORY; + } else { + return remote ? LocalizationType.REMOTE_FILE : LocalizationType.LOCAL_FILE; + } + } } } diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/LocalizerTest.java b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/LocalizerTest.java new file mode 100644 index 0000000000..db2f54ecc0 --- /dev/null +++ b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/LocalizerTest.java @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.yarnservice.utils; + +import com.google.common.collect.Lists; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.api.records.ConfigFile.TypeEnum; +import org.apache.hadoop.yarn.service.api.records.Configuration; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.submarine.FileUtilitiesForTests; +import org.apache.submarine.client.cli.param.Localization; +import org.apache.submarine.client.cli.param.runjob.RunJobParameters; +import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager; +import org.apache.submarine.server.submitter.yarnservice.FileSystemOperations; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LocalizerTest { + + public static final String DEFAULT_TEMPFILE = "testFile"; + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Mock + private RemoteDirectoryManager remoteDirectoryManager; + + @Mock + private FileSystemOperations fsOperations; + + @Mock + private Service service; + + private FileUtilitiesForTests fileUtils = new FileUtilitiesForTests(); + + private void setupService() { + Configuration conf = new Configuration(); + conf.setFiles(Lists.newArrayList()); + when(service.getConfiguration()).thenReturn(conf); + } + + private Localizer createLocalizerWithLocalizations( + Localization... localizations) { + RunJobParameters parameters = mock(RunJobParameters.class); + when(parameters.getLocalizations()) + .thenReturn(Lists.newArrayList(localizations)); + return new Localizer(fsOperations, remoteDirectoryManager, parameters); + } + + @Before + public void setUp() throws IOException { + setupService(); + fileUtils.setup(); + + when(remoteDirectoryManager.getJobStagingArea(any(), anyBoolean())) + .thenReturn(new Path("stagingarea")); + } + + private void testLocalizeExistingRemoteFileInternal() throws IOException { + when(remoteDirectoryManager.isRemote(anyString())).thenReturn(true); + when(remoteDirectoryManager.existsRemoteFile(any(Path.class))) + .thenReturn(true); + + Localization localization = new Localization(); + localization.setLocalPath("."); + localization.setRemoteUri("hdfs://dummy"); + Localizer localizer = createLocalizerWithLocalizations(localization); + + localizer.handleLocalizations(service); + verify(fsOperations).validFileSize(anyString()); + } + + private String testLocalizeExistingLocalFileInternal(String localPath) + throws IOException { + File testFile = fileUtils.createFileInTempDir(DEFAULT_TEMPFILE); + + when(remoteDirectoryManager.isRemote(anyString())).thenReturn(false); + String remoteUri = testFile.getAbsolutePath(); + when(fsOperations.uploadToRemoteFile(any(Path.class), anyString())) + .thenReturn(new Path(remoteUri)); + + Localization localization = new Localization(); + localization.setLocalPath(localPath); + localization.setRemoteUri(remoteUri); + Localizer localizer = createLocalizerWithLocalizations(localization); + + localizer.handleLocalizations(service); + verify(fsOperations).validFileSize(anyString()); + + return remoteUri; + } + + @After + public void teardown() throws IOException { + fileUtils.teardown(); + } + + @Test(expected = FileNotFoundException.class) + public void testLocalizeNotExistingRemoteFile() throws IOException { + when(remoteDirectoryManager.isRemote(anyString())).thenReturn(true); + when(remoteDirectoryManager.existsRemoteFile(any(Path.class))) + .thenReturn(false); + + Localization localization = new Localization(); + localization.setRemoteUri("hdfs://dummy"); + Localizer localizer = createLocalizerWithLocalizations(localization); + + localizer.handleLocalizations(service); + } + + @Test(expected = FileNotFoundException.class) + public void testLocalizeNotExistingLocalFile() throws IOException { + when(remoteDirectoryManager.isRemote(anyString())).thenReturn(false); + + Localization localization = new Localization(); + localization.setRemoteUri("file://dummy"); + Localizer localizer = createLocalizerWithLocalizations(localization); + + localizer.handleLocalizations(service); + } + + @Test(expected = IOException.class) + public void testLocalizeExistingRemoteFileInvalidFileSize() + throws IOException { + doThrow(IOException.class).when(fsOperations).validFileSize(anyString()); + testLocalizeExistingRemoteFileInternal(); + } + + @Test(expected = IOException.class) + public void testLocalizeExistingLocalFileInvalidFileSize() + throws IOException { + doThrow(IOException.class).when(fsOperations).validFileSize(anyString()); + testLocalizeExistingLocalFileInternal("."); + } + + @Test + public void testLocalizeExistingRemoteFile() throws IOException { + testLocalizeExistingRemoteFileInternal(); + verify(fsOperations, never()).uploadToRemoteFile(any(Path.class), + anyString()); + verify(fsOperations, never()).deleteFiles(anyString()); + + List files = service.getConfiguration().getFiles(); + assertEquals(1, files.size()); + + ConfigFile configFile = files.get(0); + assertEquals(TypeEnum.STATIC, configFile.getType()); + assertEquals("hdfs://dummy", configFile.getSrcFile()); + } + + @Test + public void testLocalizeExistingLocalFile() throws IOException { + String remoteUri = testLocalizeExistingLocalFileInternal("."); + verify(fsOperations, never()).deleteFiles(anyString()); + verify(fsOperations).uploadToRemoteFile(any(Path.class), eq(remoteUri)); + + List files = service.getConfiguration().getFiles(); + assertEquals(1, files.size()); + + ConfigFile configFile = files.get(0); + assertEquals(TypeEnum.STATIC, configFile.getType()); + assertEquals(remoteUri, configFile.getSrcFile()); + assertEquals("testFile", configFile.getDestFile()); + } + + @Test + public void testLocalizeExistingLocalFile2() throws IOException { + String remoteUri = testLocalizeExistingLocalFileInternal("./"); + verify(fsOperations, never()).deleteFiles(anyString()); + verify(fsOperations).uploadToRemoteFile(any(Path.class), eq(remoteUri)); + + List files = service.getConfiguration().getFiles(); + assertEquals(1, files.size()); + + ConfigFile configFile = files.get(0); + assertEquals(TypeEnum.STATIC, configFile.getType()); + assertEquals(remoteUri, configFile.getSrcFile()); + assertEquals("testFile", configFile.getDestFile()); + } + + @Test + public void testLocalizeExistingLocalFileAbsolute() throws IOException { + String remoteUri = + testLocalizeExistingLocalFileInternal("/dummydir/dummyfile"); + verify(fsOperations, never()).deleteFiles(anyString()); + verify(fsOperations).uploadToRemoteFile(any(Path.class), eq(remoteUri)); + + List files = service.getConfiguration().getFiles(); + assertEquals(1, files.size()); + + ConfigFile configFile = files.get(0); + assertEquals(TypeEnum.STATIC, configFile.getType()); + assertEquals(remoteUri, configFile.getSrcFile()); + assertEquals("dummyfile", configFile.getDestFile()); + + assertEquals(1, service.getConfiguration().getEnv().size()); + String dockerMounts = service.getConfiguration().getEnv() + .get(EnvironmentUtilities.ENV_DOCKER_MOUNTS_FOR_CONTAINER_RUNTIME); + assertEquals("dummyfile:/dummydir/dummyfile:rw", dockerMounts); + } + + @Test(expected = IllegalStateException.class) + public void testLocalizeExistingRemoteDirectoryNoUnderscoreInName() + throws IOException { + when(remoteDirectoryManager.isDir(anyString())).thenReturn(true); + when(remoteDirectoryManager.isRemote(anyString())).thenReturn(true); + when(remoteDirectoryManager.existsRemoteFile(any(Path.class))) + .thenReturn(true); + String remoteUri = "hdfs://remotedir1/remotedir2"; + when(fsOperations.uploadToRemoteFile(any(Path.class), anyString())) + .thenReturn(new Path(remoteUri)); + when(fsOperations.downloadAndZip(anyString(), anyString(), eq(true))) + .thenReturn("remotedir2.zip"); + + Localization localization = new Localization(); + localization.setLocalPath("remotedir2"); + localization.setRemoteUri(remoteUri); + Localizer localizer = createLocalizerWithLocalizations(localization); + + localizer.handleLocalizations(service); + verify(fsOperations).validFileSize(anyString()); + } + + @Test + public void testLocalizeExistingRemoteDirectory() throws IOException { + when(remoteDirectoryManager.isDir(anyString())).thenReturn(true); + when(remoteDirectoryManager.isRemote(anyString())).thenReturn(true); + when(remoteDirectoryManager.existsRemoteFile(any(Path.class))) + .thenReturn(true); + String remoteUri = "hdfs://remotedir1/remotedir2"; + when(fsOperations.uploadToRemoteFile(any(Path.class), anyString())) + .thenReturn(new Path(remoteUri)); + String zipFileName = "remotedir2_221424.zip"; + when(fsOperations.downloadAndZip(anyString(), anyString(), eq(true))) + .thenReturn(zipFileName); + + Localization localization = new Localization(); + localization.setLocalPath("/remotedir2"); + localization.setRemoteUri(remoteUri); + Localizer localizer = createLocalizerWithLocalizations(localization); + + localizer.handleLocalizations(service); + + verify(fsOperations).validFileSize(anyString()); + verify(fsOperations).deleteFiles(eq(zipFileName)); + verify(fsOperations, never()).uploadToRemoteFile(any(Path.class), + eq(remoteUri)); + + List files = service.getConfiguration().getFiles(); + assertEquals(1, files.size()); + + ConfigFile configFile = files.get(0); + assertEquals(TypeEnum.ARCHIVE, configFile.getType()); + assertEquals(remoteUri, configFile.getSrcFile()); + assertEquals("remotedir2", configFile.getDestFile()); + + assertEquals(1, service.getConfiguration().getEnv().size()); + String dockerMounts = service.getConfiguration().getEnv() + .get(EnvironmentUtilities.ENV_DOCKER_MOUNTS_FOR_CONTAINER_RUNTIME); + assertEquals("remotedir2:/remotedir2:rw", dockerMounts); + } + + @Test + public void testLocalizeNonHdfsRemoteUri() throws IOException { + when(remoteDirectoryManager.isRemote(anyString())).thenReturn(true); + when(remoteDirectoryManager.existsRemoteFile(any(Path.class))) + .thenReturn(true); + String remoteUri = "remote://remotedir1/remotedir2"; + when(fsOperations.uploadToRemoteFile(any(Path.class), anyString())) + .thenReturn(new Path(remoteUri)); + String downloadedFileName = "remotedir2_221424"; + when(fsOperations.downloadAndZip(anyString(), anyString(), eq(false))) + .thenReturn(downloadedFileName); + + Localization localization = new Localization(); + localization.setLocalPath("/remotedir2"); + localization.setRemoteUri(remoteUri); + Localizer localizer = createLocalizerWithLocalizations(localization); + + localizer.handleLocalizations(service); + + verify(fsOperations).validFileSize(anyString()); + verify(fsOperations).deleteFiles(eq(downloadedFileName)); + verify(fsOperations).uploadToRemoteFile(any(Path.class), + eq(downloadedFileName)); + + List files = service.getConfiguration().getFiles(); + assertEquals(1, files.size()); + + ConfigFile configFile = files.get(0); + assertEquals(TypeEnum.STATIC, configFile.getType()); + assertEquals(remoteUri, configFile.getSrcFile()); + assertEquals("remotedir2", configFile.getDestFile()); + + assertEquals(1, service.getConfiguration().getEnv().size()); + String dockerMounts = service.getConfiguration().getEnv() + .get(EnvironmentUtilities.ENV_DOCKER_MOUNTS_FOR_CONTAINER_RUNTIME); + assertEquals("remotedir2:/remotedir2:rw", dockerMounts); + } +} From 44e91ddf07b5b05e0428634e6fac4e12e8b0de06 Mon Sep 17 00:00:00 2001 From: pingsutw Date: Tue, 12 Nov 2019 18:33:00 +0800 Subject: [PATCH 04/23] SUBMARINE-290. Update submainre DEFAULT_RUNTIME_CLASS ### What is this PR for? After SUBMARINE-255 merged, we rename our runtimes to ``` org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory org.apache.submarine.server.submitter.yarnservice.YarnServiceRuntimeFactory ``` but default in SubmarineConfiguration still is `org.apache.submarine.runtimes.tony.TonyRuntimeFactory` we also need to update it. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-290 ### How should this be tested? https://travis-ci.org/pingsutw/hadoop-submarine/builds/610793459 ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: pingsutw Closes #90 from pingsutw/SUBMARINE-290 and squashes the following commits: ecbc9c2 [pingsutw] SUBMARINE-290. Update submainre DEFAULT_RUNTIME_CLASS --- docs/helper/QuickStart.md | 6 +++--- .../commons/runtime/conf/SubmarineConfiguration.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/helper/QuickStart.md b/docs/helper/QuickStart.md index b7aa2151fb..45074751f7 100644 --- a/docs/helper/QuickStart.md +++ b/docs/helper/QuickStart.md @@ -51,7 +51,7 @@ Once the applicable runtime is chosen and environment is ready, a `submarine.xml |Configuration Name | Description | |:---- |:---- | -| `submarine.runtime.class` | "org.apache.submarine.runtimes.tony.TonyRuntimeFactory" or "org.apache.submarine.runtimes.yarnservice.YarnServiceRuntimeFactory" | +| `submarine.runtime.class` | "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory" or "org.apache.submarine.server.submitter.yarnservice.YarnServiceRuntimeFactory" |
@@ -61,9 +61,9 @@ A sample `submarine.xml` is here: submarine.runtime.class - org.apache.submarine.runtimes.tony.TonyRuntimeFactory + org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory diff --git a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java index 9daefd62c8..b51446b07f 100644 --- a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java +++ b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java @@ -61,7 +61,7 @@ public SubmarineConfiguration(Configuration configuration, public static final String RUNTIME_CLASS = PREFIX + "runtime.class"; public static final String DEFAULT_RUNTIME_CLASS = - "org.apache.submarine.runtimes.tony.TonyRuntimeFactory"; + "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"; public void setSubmarineRuntimeClass(String runtimeClass) { set(RUNTIME_CLASS, runtimeClass); From 02c070a41d4af0027249e5ddf09642d0e0d492fe Mon Sep 17 00:00:00 2001 From: Zhankun Tang Date: Wed, 13 Nov 2019 13:19:39 +0800 Subject: [PATCH 05/23] SUBMARINE-267. Initial job server which defines REST skeleton ### What is this PR for? The initial job server which defines REST skeleton ### What type of PR is it? Feature ### Todos * [1] - Fix the YAML unit test case issue * [2] - Populate the REST API with integration to yarn-submitter or k8s-submitter * [3] - Add Swagger to easy access the API spec ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-267 ### How should this be tested? CI unit test is enough for the REST API before we integrate with submitter ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Are there breaking changes for older versions? No * Does this needs documentation? No Author: Zhankun Tang Closes #87 from tangzhankun/submarine-267 and squashes the following commits: 5303d3f [Zhankun Tang] Add submarine server core test in Travis CI a355ff0 [Zhankun Tang] fix dependency issue for hadoop 3.1 and 3.2 441317a [Zhankun Tang] Remove workbench server dependency from submarine server-core ae3cd5f [Zhankun Tang] Fix wrong test case name and inproper log method d97cdb7 [Zhankun Tang] SUBMARINE-267. Initial version of job server which defines job spec and REST skelon. --- .travis.yml | 9 +- pom.xml | 3 +- .../commons/utils/SubmarineConfiguration.java | 34 ++- submarine-server/pom.xml | 68 ++++++ submarine-server/server-core/pom.xml | 102 ++++++++ .../apache/submarine/jobserver/JobServer.java | 165 +++++++++++++ .../submarine/jobserver/rest/api/JobApi.java | 93 ++++++++ .../jobserver/rest/dao/Component.java | 69 ++++++ .../jobserver/rest/dao/EnvVaraible.java | 53 +++++ .../rest/dao/JsonExclusionStrategy.java | 33 +++ .../jobserver/rest/dao/JsonResponse.java | 217 ++++++++++++++++++ .../jobserver/rest/dao/MLJobSpec.java | 177 ++++++++++++++ .../jobserver/rest/dao/RestConstants.java | 29 +++ .../rest/provider/YamlEntityProvider.java | 91 ++++++++ .../src/main/resources/log4j.properties | 17 ++ .../server-core/src/test/java/JobApiTest.java | 146 ++++++++++++ 16 files changed, 1300 insertions(+), 6 deletions(-) create mode 100644 submarine-server/server-core/pom.xml create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java create mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java create mode 100644 submarine-server/server-core/src/main/resources/log4j.properties create mode 100644 submarine-server/server-core/src/test/java/JobApiTest.java diff --git a/.travis.yml b/.travis.yml index 037c9cb2be..af2a24e01b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ env: global: # submarine core does not required by workbench-server integration tests # If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE - - EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-submitter/submitter-k8s" + - EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-submitter/submitter-k8s,!submarine-server/server-core" - EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web,!submarine-workbench/workbench-server" - EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter"" - EXCLUDE_SUBMODULE_TONY="!submodules/tony,!submodules/tony/tony-mini,!submodules/tony/tony-core,!submodules/tony/tony-proxy,!submodules/tony/tony-portal,!submodules/tony/tony-azkaban,!submodules/tony/tony-cli" @@ -160,6 +160,13 @@ matrix: - npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js env: NAME="Build workbench-web-ng" + + # Test submarine-server + - language: java + jdk: "openjdk8" + dist: xenial + env: NAME="Test submarine-server" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl submarine-server/server-core" TEST_PROJECTS="" + install: - mvn --version - echo "[$NAME] > mvn $BUILD_FLAG $MODULES $PROFILE -B" diff --git a/pom.xml b/pom.xml index df70ae0e1f..8936bae9dd 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,7 @@ 1.16 4.4.4 4.5.2 - 2.5 + 2.6 3.4 2.5 1.5 @@ -124,6 +124,7 @@ 2.1.1 6.0.1 + 2.27 diff --git a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java index 2ddb50c402..44cea260d1 100644 --- a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java +++ b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java @@ -113,11 +113,11 @@ public static synchronized SubmarineConfiguration create() { } } - LOG.info("Server Host: " + conf.getServerAddress()); + LOG.info("Workbench server Host: " + conf.getServerAddress()); if (conf.useSsl() == false) { - LOG.info("Server Port: " + conf.getServerPort()); + LOG.info("Workbench server Port: " + conf.getServerPort()); } else { - LOG.info("Server SSL Port: " + conf.getServerSslPort()); + LOG.info("Workbench server SSL Port: " + conf.getServerSslPort()); } return conf; @@ -127,14 +127,26 @@ public String getServerAddress() { return getString(ConfVars.SERVER_ADDR); } + public String getJobServerAddress() { + return getString(ConfVars.JOB_SERVER_ADDR); + } + public boolean useSsl() { return getBoolean(ConfVars.SERVER_SSL); } + public boolean isJobServerSslEnabled() { + return getBoolean(ConfVars.JOB_SERVER_SSL); + } + public int getServerPort() { return getInt(ConfVars.SERVER_PORT); } + public int getJobServerPort() { + return getInt(ConfVars.JOB_SERVER_PORT); + } + @VisibleForTesting public void setServerPort(int port) { properties.put(ConfVars.SERVER_PORT.getVarName(), String.valueOf(port)); @@ -144,6 +156,14 @@ public int getServerSslPort() { return getInt(ConfVars.SERVER_SSL_PORT); } + public int getJobServerSslPort() { + return getInt(ConfVars.JOB_SERVER_SSL_PORT); + } + + public String getJobServerUrlPrefix() { + return getString(ConfVars.JOB_SERVER_REST_URL_PREFIX); + } + public String getKeyStorePath() { String path = getString(ConfVars.SSL_KEYSTORE_PATH); return path; @@ -419,7 +439,13 @@ public enum ConfVars { JDBC_PASSWORD("jdbc.password", "password"), WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE( "workbench.websocket.max.text.message.size", "1024000"), - WORKBENCH_WEB_WAR("workbench.web.war", "submarine-workbench/workbench-web/dist"); + WORKBENCH_WEB_WAR("workbench.web.war", "submarine-workbench/workbench-web/dist"), + // submarine job server settings + JOB_SERVER_SSL("job.server.ssl", false), + JOB_SERVER_SSL_PORT("job.server.ssl.port", 8443), + JOB_SERVER_ADDR("job.server.port", "0.0.0.0"), + JOB_SERVER_PORT("job.server.port", 8765), + JOB_SERVER_REST_URL_PREFIX("job.server.rest.prefix", "/*"); private String varName; @SuppressWarnings("rawtypes") diff --git a/submarine-server/pom.xml b/submarine-server/pom.xml index 0e7629a2c2..a367703ff5 100644 --- a/submarine-server/pom.xml +++ b/submarine-server/pom.xml @@ -35,6 +35,7 @@ server-submitter + server-core @@ -43,6 +44,73 @@ org.apache.submarine commons-runtime ${project.version} + + + commons-logging + commons-logging + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + + + com.fasterxml.jackson.core + jackson-databind + + + org.eclipse.jetty + jetty-util + + + + + org.apache.submarine + commons-utils + ${project.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + org.glassfish.jersey.test-framework + jersey-test-framework-core + ${jersey.test-framework} + + + javax.servlet + javax.servlet-api + + + test + + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + ${jersey.test-framework} + + + javax.servlet + javax.servlet-api + + + test + + + com.google.code.gson + gson + ${gson.version} diff --git a/submarine-server/server-core/pom.xml b/submarine-server/server-core/pom.xml new file mode 100644 index 0000000000..84c075e266 --- /dev/null +++ b/submarine-server/server-core/pom.xml @@ -0,0 +1,102 @@ + + + + + + submarine-server + org.apache.submarine + 0.3.0-SNAPSHOT + + 4.0.0 + + server-core + Submarine: Submarine Server Core + + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-servlet + ${jetty.version} + + + org.glassfish.jersey.containers + jersey-container-servlet-core + ${jersey.version} + + + + org.glassfish.jersey.core + jersey-server + ${jersey.version} + + + + org.glassfish.jersey.inject + jersey-hk2 + ${jersey.version} + + + + org.glassfish.jersey.media + jersey-media-json-jackson + ${jersey.version} + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + org.apache.submarine + commons-utils + + + org.apache.submarine + commons-runtime + + + org.yaml + snakeyaml + + + org.glassfish.jersey.test-framework + jersey-test-framework-core + test + + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + test + + + com.google.code.gson + gson + + + \ No newline at end of file diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java new file mode 100644 index 0000000000..298081e31c --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.submarine.jobserver; + + +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.eclipse.jetty.util.thread.ThreadPool; +import org.glassfish.jersey.servlet.ServletContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.submarine.commons.utils.SubmarineConfiguration; + + +/** + * The ml job server. It will load the classes in rest package when + * bootstrap with related configurable settings. + * */ +public class JobServer { + + private static final Logger LOG = LoggerFactory.getLogger(JobServer.class); + + private SubmarineConfiguration conf = SubmarineConfiguration.create(); + + private Server jobServer; + + public void start() { + ServletContextHandler context = new + ServletContextHandler(ServletContextHandler.SESSIONS); + context.setContextPath("/"); + setupServer(conf);; + jobServer.setHandler(context); + + // Job API servlet + ServletHolder apiServlet = context.addServlet(ServletContainer.class, + conf.getJobServerUrlPrefix()); + apiServlet.setInitOrder(1); + apiServlet.setInitParameter("jersey.config.server.provider.packages", + "org.apache.submarine.jobserver.rest"); + + try { + jobServer.start(); + LOG.info("Submarine job server started"); + jobServer.join(); + } catch (Exception e) { + LOG.error("Submarine job server failed to start"); + e.printStackTrace(); + } finally { + jobServer.destroy(); + LOG.info("Submarine job server stopped"); + } + } + + public static void main(String[] args) throws Exception { + new JobServer().start(); + } + + private Server setupServer(SubmarineConfiguration conf) { + ThreadPool threadPool = + new QueuedThreadPool( + conf.getInt(SubmarineConfiguration + .ConfVars.SERVER_JETTY_THREAD_POOL_MAX), + conf.getInt(SubmarineConfiguration + .ConfVars.SERVER_JETTY_THREAD_POOL_MIN), + conf.getInt(SubmarineConfiguration + .ConfVars.SERVER_JETTY_THREAD_POOL_TIMEOUT)); + jobServer = new Server(threadPool); + ServerConnector connector; + + if (conf.isJobServerSslEnabled()) { + LOG.debug("Enabling SSL for submarine job server on port " + + conf.getServerSslPort()); + HttpConfiguration httpConfig = new HttpConfiguration(); + httpConfig.setSecureScheme("https"); + httpConfig.setSecurePort(conf.getServerSslPort()); + httpConfig.setOutputBufferSize(32768); + httpConfig.setResponseHeaderSize(8192); + httpConfig.setSendServerVersion(true); + + HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig); + SecureRequestCustomizer src = new SecureRequestCustomizer(); + httpsConfig.addCustomizer(src); + + connector = new ServerConnector( + jobServer, + new SslConnectionFactory(getSslContextFactory(conf), + HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(httpsConfig)); + } else { + connector = new ServerConnector(jobServer); + } + + configureRequestHeaderSize(conf, connector); + // Set some timeout options to make debugging easier. + int timeout = 1000 * 30; + connector.setIdleTimeout(timeout); + connector.setSoLingerTime(-1); + connector.setHost(conf.getJobServerAddress()); + if (conf.useSsl()) { + connector.setPort(conf.getJobServerSslPort()); + } else { + connector.setPort(conf.getJobServerPort()); + } + + jobServer.addConnector(connector); + return jobServer; + } + + private static SslContextFactory getSslContextFactory( + SubmarineConfiguration conf) { + SslContextFactory sslContextFactory = new SslContextFactory(); + + // Set keystore + sslContextFactory.setKeyStorePath(conf.getKeyStorePath()); + sslContextFactory.setKeyStoreType(conf.getKeyStoreType()); + sslContextFactory.setKeyStorePassword(conf.getKeyStorePassword()); + sslContextFactory.setKeyManagerPassword(conf.getKeyManagerPassword()); + + if (conf.useClientAuth()) { + sslContextFactory.setNeedClientAuth(conf.useClientAuth()); + + // Set truststore + sslContextFactory.setTrustStorePath(conf.getTrustStorePath()); + sslContextFactory.setTrustStoreType(conf.getTrustStoreType()); + sslContextFactory.setTrustStorePassword(conf.getTrustStorePassword()); + } + + return sslContextFactory; + } + + private static void configureRequestHeaderSize( + SubmarineConfiguration conf, ServerConnector connector) { + HttpConnectionFactory cf = + (HttpConnectionFactory) connector + .getConnectionFactory(HttpVersion.HTTP_1_1.toString()); + int requestHeaderSize = conf.getJettyRequestHeaderSize(); + cf.getHttpConfiguration().setRequestHeaderSize(requestHeaderSize); + } + +} diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java new file mode 100644 index 0000000000..8b842f49d1 --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.submarine.jobserver.rest.api; + +import org.apache.submarine.jobserver.rest.dao.JsonResponse; +import org.apache.submarine.jobserver.rest.dao.MLJobSpec; +import org.apache.submarine.jobserver.rest.dao.RestConstants; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +/** + * ML job rest API v1. It can accept MLJobSpec to create a job. + * To create a job: + * POST /api/v1/jobs + * + * To list the jobs + * GET /api/v1/jobs + * + * To get a specific job + * GET /api/v1/jobs/{id} + * + * To delete a job by id + * DELETE /api/v1/jobs/{id} + * */ +@Path(RestConstants.V1 + "/" + RestConstants.JOBS) +@Produces({MediaType.APPLICATION_JSON + "; " + RestConstants.CHARSET_UTF8}) +public class JobApi { + + // A ping test to verify the job server is up. + @Path(RestConstants.PING) + @GET + @Consumes(MediaType.APPLICATION_JSON) + public Response ping() { + return new JsonResponse.Builder(Response.Status.OK) + .success(true).result("Pong").build(); + } + + @POST + @Consumes({RestConstants.MEDIA_TYPE_YAML, MediaType.APPLICATION_JSON}) + public Response submitJob(MLJobSpec jobSpec) { + // Submit the job spec through submitter + return new JsonResponse.Builder(Response.Status.ACCEPTED) + .success(true).result(jobSpec).build(); + } + + @GET + @Path("{" + RestConstants.JOB_ID + "}") + public Response listJob(@PathParam(RestConstants.JOB_ID) String id) { + // Query the job status though submitter + return new JsonResponse.Builder(Response.Status.OK) + .success(true).result(id).build(); + } + + @GET + public Response listAllJob() { + // Query all the job status though submitter + return new JsonResponse.Builder(Response.Status.OK) + .success(true).build(); + } + + @DELETE + @Path("{" + RestConstants.JOB_ID + "}") + public Response deleteJob(@PathParam(RestConstants.JOB_ID) String id) { + // Delete the job though submitter + return new JsonResponse.Builder(Response.Status.OK) + .success(true).result(id).build(); + } + +} diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java new file mode 100644 index 0000000000..a2c1458d52 --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.jobserver.rest.dao; + + +/** + * One component contains role, count and resources. + * The role name could be tensorflow ps, Pytorch master or tensorflow worker + * The count is the count of the role instance + * The resource is the memory/vcore/gpu resource strings in the format: + * "memory=2048M,vcore=4,nvidia.com/gpu=1" + */ + +public class Component { + + public String getRole() { + return role; + } + + public void setRole(String role) { + this.role = role; + } + + public String getCount() { + return count; + } + + public void setCount(String count) { + this.count = count; + } + + public String getResources() { + return resources; + } + + public void setResources(String resources) { + this.resources = resources; + } + + String role; + String count; + String resources; + + public Component() {} + + public Component(String r, String ct, String res) { + this.count = ct; + this.role = r; + this.resources = res; + } + +} diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java new file mode 100644 index 0000000000..24a4ae2aca --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.jobserver.rest.dao; + +// A process level environment variable. +public class EnvVaraible { + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + String key; + String value; + + public EnvVaraible() {} + + public EnvVaraible(String k, String v) { + this.key = k; + this.value = v; + } + + + +} diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java new file mode 100644 index 0000000000..c24e413429 --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.jobserver.rest.dao; + +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; + +public class JsonExclusionStrategy implements ExclusionStrategy { + public boolean shouldSkipClass(Class arg0) { + return false; + } + + public boolean shouldSkipField(FieldAttributes f) { + return false; + } +} diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java new file mode 100644 index 0000000000..46726cf24b --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.jobserver.rest.dao; + +import com.google.common.annotations.VisibleForTesting; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.TypeAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.NewCookie; +import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Json response builder. + * + * @param can be an object or a ListResult + */ +public class JsonResponse { + private static final Logger LOG = LoggerFactory.getLogger(JsonResponse.class); + + private final javax.ws.rs.core.Response.Status status; + private final int code; + private final Boolean success; + private final String message; + private final T result; + private final transient ArrayList cookies; + private final transient boolean pretty = false; + private final Map attributes; + + private static Gson safeGson = null; + + private static final String CGLIB_PROPERTY_PREFIX = "\\$cglib_prop_"; + + private JsonResponse(Builder builder) { + this.status = builder.status; + this.code = builder.code; + this.success = builder.success; + this.message = builder.message; + this.attributes = builder.attributes; + this.result = (T) builder.result; + this.cookies = builder.cookies; + } + + public T getResult() { + return result; + } + + public Boolean getSuccess() { + return success; + } + + @VisibleForTesting + public Map getAttributes() { + return attributes; + } + + public static class Builder { + private javax.ws.rs.core.Response.Status status; + private int code; + private Boolean success; + private String message; + private T result; + private Map attributes = new HashMap<>(); + private transient ArrayList cookies; + private transient boolean pretty = false; + + public Builder(javax.ws.rs.core.Response.Status status) { + this.status = status; + this.code = status.getStatusCode(); + } + + public Builder(int code) { + this.code = code; + } + + public Builder attribute(String key, Object value) { + this.attributes.put(key, value); + return this; + } + + public Builder success(Boolean success) { + this.success = success; + return this; + } + + public Builder message(String message) { + this.message = message; + return this; + } + + public Builder result(T result) { + this.result = result; + return this; + } + + public Builder code(int code) { + this.code = code; + return this; + } + + public Builder cookies(ArrayList newCookies) { + if (cookies == null) { + cookies = new ArrayList<>(); + } + cookies.addAll(newCookies); + return this; + } + + public javax.ws.rs.core.Response build() { + JsonResponse jsonResponse = new JsonResponse(this); + return jsonResponse.build(); + } + } + + @Override + public String toString() { + if (safeGson == null) { + GsonBuilder gsonBuilder = new GsonBuilder(); + if (pretty) { + gsonBuilder.setPrettyPrinting(); + } + gsonBuilder.setExclusionStrategies(new JsonExclusionStrategy()); + + // Trick to get the DefaultDateTypeAdatpter instance + // Create a first instance a Gson + Gson gson = gsonBuilder.setDateFormat("yyyy-MM-dd HH:mm:ss").create(); + + // Get the date adapter + TypeAdapter dateTypeAdapter = gson.getAdapter(Date.class); + + // Ensure the DateTypeAdapter is null safe + TypeAdapter safeDateTypeAdapter = dateTypeAdapter.nullSafe(); + + safeGson = new GsonBuilder() + .registerTypeAdapter(Date.class, safeDateTypeAdapter) + .serializeNulls().create(); + } + +// boolean haveDictAnnotation = false; +// try { +// if (null != getResult()) { +// haveDictAnnotation = DictAnnotation.parseDictAnnotation(getResult()); +// } +// } catch (Exception e) { +// LOG.error(e.getMessage(), e); +// } + + String json = safeGson.toJson(this); +// if (haveDictAnnotation) { +// json = json.replaceAll(CGLIB_PROPERTY_PREFIX, ""); +// } + + return json; + } + + private synchronized javax.ws.rs.core.Response build() { + Response.ResponseBuilder r = javax.ws.rs.core.Response.status(status).entity(this.toString()); + if (cookies != null) { + for (NewCookie nc : cookies) { + r.cookie(nc); + } + } + return r.build(); + } + + // list result type response + // Used to return a list of records + public static class ListResult { + private List records; + private long total; + + public ListResult(List records, long total) { + this.records = records; + this.total = total; + } + + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } + + public long getTotal() { + return total; + } + + public void setTotal(long total) { + this.total = total; + } + } +} diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java new file mode 100644 index 0000000000..26f7fa823a --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.jobserver.rest.dao; + +/** + * The machine learning job spec the submarine job server can accept. + * */ +public class MLJobSpec { + + String apiVersion; + // The engine type the job will use, can be tensorflow or pytorch + String type; + // The engine version, not image version/tag. For instance, tensorflow v1.13 + String version; + /** + * Advanced property. The RM this job will submit to, k8s or yarn. + * Could be the path of yarn’s config file or k8s kubeconfig file + * This should be settled when deploy submarine job server. + */ + String rmConfig; + + /** + * Advanced property. The image should be inferred from type and version. + * The normal user should not set this. + * */ + String dockerImage; + + // The process aware environment variable + EnvVaraible[] envVars; + + // The components this cluster will consists. + Component[] components; + + // The user id who submit job + String uid; + + /** + * The queue this job will submitted to. + * In YARN, we call it queue. In k8s, no such concept. + * It could be namespace’s name. + */ + String queue; + + /** + * The user-specified job name for easy search + * */ + String name; + + /** + * This could be file/directory which contains multiple python scripts. + * We should solve dependencies distribution in k8s or yarn. + * Or we could build images for each projects before submitting the job + * */ + String projects; + + /** + * The command uses to start the job. This is very job-specific. + * We assume the cmd is the same for all containers in a cluster + * */ + String cmd; + + public MLJobSpec() {} + + public String getUid() { + return uid; + } + + public void setUid(String uid) { + this.uid = uid; + } + + public String getQueue() { + return queue; + } + + public void setQueue(String queue) { + this.queue = queue; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getProjects() { + return projects; + } + + public void setProjects(String projects) { + this.projects = projects; + } + public String getCmd() { + return cmd; + } + + public void setCmd(String cmd) { + this.cmd = cmd; + } + + public Component[] getComponents() { + return components; + } + + public void setComponents( + Component[] components) { + this.components = components; + } + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getRmConfig() { + return rmConfig; + } + + public void setRmConfig(String rmConfig) { + this.rmConfig = rmConfig; + } + + public String getDockerImage() { + return dockerImage; + } + + public void setDockerImage(String dockerImage) { + this.dockerImage = dockerImage; + } + + public EnvVaraible[] getEnvVars() { + return envVars; + } + + public void setEnvVars(EnvVaraible[] envVars) { + this.envVars = envVars; + } + + public String getApiVersion() { + return apiVersion; + } + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + +} diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java new file mode 100644 index 0000000000..a33a9314e1 --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.jobserver.rest.dao; + +public class RestConstants { + public final static String V1 = "v1"; + public final static String JOBS = "jobs"; + public final static String JOB_ID = "id"; + public final static String PING = "ping"; + public final static String MEDIA_TYPE_YAML = "application/yaml"; + public final static String CHARSET_UTF8 = "charset=utf-8"; +} diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java new file mode 100644 index 0000000000..3343cc89e2 --- /dev/null +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.jobserver.rest.provider; + +import org.yaml.snakeyaml.Yaml; + +import javax.ws.rs.Consumes; +import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyReader; +import javax.ws.rs.ext.MessageBodyWriter; +import javax.ws.rs.ext.Provider; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.util.Scanner; + +@Provider +@Consumes({"application/yaml", MediaType.TEXT_PLAIN}) +@Produces({"application/yaml", MediaType.TEXT_PLAIN}) +public class YamlEntityProvider implements MessageBodyWriter, MessageBodyReader { + + @Override + public boolean isReadable(Class type, Type genericType, + Annotation[] annotations, + MediaType mediaType) { + return true; + } + + @Override + public T readFrom(Class type, Type genericType, Annotation[] annotations, + MediaType mediaType, + MultivaluedMap httpHeaders, InputStream entityStream) + throws WebApplicationException { + Yaml yaml = new Yaml(); + T t = yaml.loadAs(toString(entityStream), type); + return t; + } + + public static String toString(InputStream inputStream) { + return new Scanner(inputStream, "UTF-8") + .useDelimiter("\\A").next(); + } + + @Override + public boolean isWriteable(Class type, Type genericType, + Annotation[] annotations, + MediaType mediaType) { + return true; + } + + @Override + public long getSize(T t, Class type, Type genericType, + Annotation[] annotations, + MediaType mediaType) { + return -1; + } + + @Override + public void writeTo(T t, Class type, Type genericType, + Annotation[] annotations, + MediaType mediaType, MultivaluedMap httpHeaders, + OutputStream entityStream) throws IOException, WebApplicationException { + Yaml yaml = new Yaml(); + OutputStreamWriter writer = new OutputStreamWriter(entityStream); + yaml.dump(t, writer); + writer.close(); + } +} diff --git a/submarine-server/server-core/src/main/resources/log4j.properties b/submarine-server/server-core/src/main/resources/log4j.properties new file mode 100644 index 0000000000..55e02b6d20 --- /dev/null +++ b/submarine-server/server-core/src/main/resources/log4j.properties @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +log4j.rootLogger = info, stdout + +log4j.appender.stdout = org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target = System.out +log4j.appender.stdout.layout = org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n diff --git a/submarine-server/server-core/src/test/java/JobApiTest.java b/submarine-server/server-core/src/test/java/JobApiTest.java new file mode 100644 index 0000000000..6a7ef12bcd --- /dev/null +++ b/submarine-server/server-core/src/test/java/JobApiTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.google.gson.Gson; +import org.apache.submarine.jobserver.rest.dao.JsonResponse; +import org.apache.submarine.jobserver.rest.dao.RestConstants; +import org.apache.submarine.jobserver.rest.api.JobApi; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.JerseyTest; +import org.junit.Test; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import static org.junit.Assert.assertEquals; + +public class JobApiTest extends JerseyTest { + + @Override + protected Application configure() { + return new ResourceConfig(JobApi.class); + } + + @Test + public void testJobServerPing() { + String str = "Pong"; + Response response = target(RestConstants.V1 + "/" + + RestConstants.JOBS + "/" + RestConstants.PING) + .request() + .get(); + Gson gson = new Gson(); + JsonResponse r = gson.fromJson(response.readEntity(String.class), + JsonResponse.class); + assertEquals("Response code should be 200 ", + Response.Status.OK.getStatusCode(), response.getStatus()); + assertEquals("Response message should be " + str, + str, r.getResult().toString()); + } + + // Test job created with correct JSON input + @Test + public void testCreateJobWhenJsonInputIsCorrectThenResponseCodeAccepted() { + String jobSpec = "{\"type\": \"tensorflow\", \"version\":\"v1.13\"}"; + Response response = target(RestConstants.V1 + "/" + RestConstants.JOBS) + .request() + .post(Entity.entity(jobSpec, MediaType.APPLICATION_JSON)); + + assertEquals("Http Response should be 202 ", + Response.Status.ACCEPTED.getStatusCode(), response.getStatus()); + } + + // Test job created with incorrect JSON input + @Test + public void testCreateJobWhenJsonInputIsWrongThenResponseCodeBadRequest() { + String jobSpec = "{\"ttype\": \"tensorflow\", \"version\":\"v1.13\"}"; + Response response = target(RestConstants.V1 + "/" + RestConstants.JOBS) + .request() + .post(Entity.entity(jobSpec, MediaType.APPLICATION_JSON)); + + assertEquals("Http Response should be 400 ", + Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); + } + + // Test get job list + @Test + public void testGetJobList() { + Response response = target(RestConstants.V1 + "/" + RestConstants.JOBS) + .request() + .get(); + + assertEquals("Http Response should be 200 ", + Response.Status.OK.getStatusCode(), response.getStatus()); + } + + // Test get job by id + @Test + public void testGetJobById() { + String jobId = "job1"; + Response response = target(RestConstants.V1 + "/" + + RestConstants.JOBS + "/" + jobId) + .request() + .get(); + Gson gson = new Gson(); + JsonResponse r = gson.fromJson(response.readEntity(String.class), + JsonResponse.class); + assertEquals("Http Response should be 200 ", + Response.Status.OK.getStatusCode(), response.getStatus()); + assertEquals("Job id should be " + jobId, + jobId, r.getResult().toString()); + } + + // Test delete job by id + @Test + public void testDeleteJobById() { + String jobId = "job1"; + Response response = target(RestConstants.V1 + "/" + + RestConstants.JOBS + "/" + jobId) + .request() + .delete(); + Gson gson = new Gson(); + JsonResponse r = gson.fromJson(response.readEntity(String.class), + JsonResponse.class); + assertEquals("Http Response should be 200 ", + Response.Status.OK.getStatusCode(), response.getStatus()); + assertEquals("Deleted job id should be " + jobId, + jobId, r.getResult().toString()); + } + + /** + * FiXME. The manual YAML test with postman works but failed here. + * We need to figure out why the YAML entity provider not work in this test. + * */ + @Test + public void testCreateJobWhenYamlInputIsCorrectThenResponseCodeAccepted() { +// Client client = ClientBuilder.newBuilder() +// .register(new YamlEntityProvider<>()).build(); +// this.setClient(client); +// String jobSpec = "type: tf"; +// Response response = target(RestConstants.V1 + "/" +// + RestConstants.JOBS + "/" + "test") +// .request() +// .put(Entity.entity(jobSpec, "application/yaml")); +// +// assertEquals("Http Response should be 202 ", +// Response.Status.ACCEPTED.getStatusCode(), response.getStatus()); + } + +} From 7a04419a96e403548326a58c16f98184f496492e Mon Sep 17 00:00:00 2001 From: Zac Zhou Date: Wed, 13 Nov 2019 18:15:26 +0800 Subject: [PATCH 06/23] SUBMARINE-276. Submarine runtime class config should be defined in the new configuration class. ### What is this PR for? There are two SubmarineConfiguration classes in the submarine project. Let's combine them into the new configuration class org.apache.submarine.commons.utils.SubmarineConfiguration. ### What type of PR is it? Improvement ### What is the Jira issue? https://issues.apache.org/jira/browse/SUBMARINE-276 ### How should this be tested? https://travis-ci.org/yuanzac/hadoop-submarine/builds/611726441?utm_medium=notification&utm_source=github_status ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? Yes * Does this needs documentation? Yes Author: Zac Zhou Closes #92 from yuanzac/topic/SUBMARINE-276 and squashes the following commits: f48ff17 [Zac Zhou] SUBMARINE-276. Submarine runtime class config should be defined in the new configuration class. --- conf/submarine-site.xml | 6 ++ conf/submarine-site.xml.template | 6 ++ dev-support/mini-submarine/README.md | 2 +- dev-support/mini-submarine/conf/bootstrap.sh | 2 +- .../{submarine.xml => submarine-site.xml} | 0 docs/helper/QuickStart.md | 4 +- pom.xml | 10 +++ .../submarine/client/cli/AbstractCli.java | 4 +- .../org/apache/submarine/client/cli/Cli.java | 2 +- .../commons/cluster/ClusterManager.java | 2 +- .../commons/cluster/ClusterMonitor.java | 2 +- .../commons/cluster/ClusterMultiNodeTest.java | 2 +- submarine-commons/commons-runtime/pom.xml | 11 +++ .../commons/runtime/ClientContext.java | 12 ++-- .../commons/runtime/RuntimeFactory.java | 10 ++- .../runtime/conf/SubmarineConfiguration.java | 69 ------------------- submarine-commons/commons-utils/pom.xml | 7 -- .../commons/utils/SubmarineConfiguration.java | 48 +++++++++---- .../apache/submarine/jobserver/JobServer.java | 2 +- .../server/submitter/yarn/YarnUtils.java | 2 +- .../yarnservice/FileSystemOperations.java | 10 +-- .../TestYarnServiceRunJobCliLocalization.java | 11 +-- .../interpreter/InterpreterProcess.java | 2 +- .../interpreter/InterpreterClusterTest.java | 2 +- .../submarine/database/MyBatisUtil.java | 4 +- .../submarine/server/WorkbenchServer.java | 10 ++- .../server/WorkbenchClusterServerTest.java | 2 +- 27 files changed, 113 insertions(+), 131 deletions(-) rename dev-support/mini-submarine/conf/{submarine.xml => submarine-site.xml} (100%) delete mode 100644 submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java diff --git a/conf/submarine-site.xml b/conf/submarine-site.xml index 8f77a54f68..a8e7794225 100755 --- a/conf/submarine-site.xml +++ b/conf/submarine-site.xml @@ -19,6 +19,12 @@ + + submarine.runtime.class + org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory + RuntimeFactory for Submarine jobs + + workbench.server.addr 0.0.0.0 diff --git a/conf/submarine-site.xml.template b/conf/submarine-site.xml.template index 8f77a54f68..a8e7794225 100755 --- a/conf/submarine-site.xml.template +++ b/conf/submarine-site.xml.template @@ -19,6 +19,12 @@ + + submarine.runtime.class + org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory + RuntimeFactory for Submarine jobs + + workbench.server.addr 0.0.0.0 diff --git a/dev-support/mini-submarine/README.md b/dev-support/mini-submarine/README.md index faaae88faa..185e54fa31 100644 --- a/dev-support/mini-submarine/README.md +++ b/dev-support/mini-submarine/README.md @@ -247,7 +247,7 @@ cd && cd spark-script && ./run_spark.sh 1. Submarine package name error - Because the package name of submarine 0.3.0 or higher has been changed from `apache.hadoop.yarn.submarine` to `apache.submarine`, So you need to set the Runtime settings in the `/usr/local/hadoop/etc/hadoop/submarine.xml` file. + Because the package name of submarine 0.3.0 or higher has been changed from `apache.hadoop.yarn.submarine` to `apache.submarine`, So you need to set the Runtime settings in the `/usr/local/hadoop/etc/hadoop/submarine-site.xml` file. ``` diff --git a/dev-support/mini-submarine/conf/bootstrap.sh b/dev-support/mini-submarine/conf/bootstrap.sh index f07c63dcee..68b2f11552 100755 --- a/dev-support/mini-submarine/conf/bootstrap.sh +++ b/dev-support/mini-submarine/conf/bootstrap.sh @@ -27,7 +27,7 @@ CONFIG_DIR="/tmp/hadoop-config" # Copy config files from volume mount -for f in slaves core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml container-executor.cfg capacity-scheduler.xml node-resources.xml resource-types.xml submarine.xml; do +for f in slaves core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml container-executor.cfg capacity-scheduler.xml node-resources.xml resource-types.xml submarine-site.xml; do if [[ -e ${CONFIG_DIR}/$f ]]; then cp ${CONFIG_DIR}/$f $HADOOP_PREFIX/etc/hadoop/$f else diff --git a/dev-support/mini-submarine/conf/submarine.xml b/dev-support/mini-submarine/conf/submarine-site.xml similarity index 100% rename from dev-support/mini-submarine/conf/submarine.xml rename to dev-support/mini-submarine/conf/submarine-site.xml diff --git a/docs/helper/QuickStart.md b/docs/helper/QuickStart.md index 45074751f7..dd5c6ac038 100644 --- a/docs/helper/QuickStart.md +++ b/docs/helper/QuickStart.md @@ -47,7 +47,7 @@ Note that if you want to quickly try Submarine on new or existing YARN cluster, For the environment setup, please check [InstallationGuide](InstallationGuide.md) or [InstallationGuideCN](InstallationGuideChineseVersion.md) -Once the applicable runtime is chosen and environment is ready, a `submarine.xml` need to be created under `$HADOOP_CONF_DIR`. To use the TonY runtime, please set below value in the submarine configuration. +Once the applicable runtime is chosen and environment is ready, a `submarine-site.xml` need to be created under `$HADOOP_CONF_DIR`. To use the TonY runtime, please set below value in the submarine configuration. |Configuration Name | Description | |:---- |:---- | @@ -55,7 +55,7 @@ Once the applicable runtime is chosen and environment is ready, a `submarine.xml
-A sample `submarine.xml` is here: +A sample `submarine-site.xml` is here: ```java diff --git a/pom.xml b/pom.xml index 8936bae9dd..82234c770e 100644 --- a/pom.xml +++ b/pom.xml @@ -246,6 +246,16 @@ commons-io ${commons-io.version} + + commons-configuration + commons-configuration + ${commons-configuration.version} + + + commons-lang + commons-lang + ${commons-lang.version} + org.apache.zookeeper zookeeper diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java index a15397b279..177bf160ab 100644 --- a/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/AbstractCli.java @@ -42,11 +42,11 @@ public abstract int run(String[] args) @Override public void setConf(Configuration conf) { - clientContext.setSubmarineConfig(conf); + clientContext.setYarnConfig(conf); } @Override public Configuration getConf() { - return clientContext.getSubmarineConfig(); + return clientContext.getYarnConfig(); } } diff --git a/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java b/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java index f13d20871d..4b8d0ad61b 100644 --- a/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java +++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java @@ -47,7 +47,7 @@ private static void printHelp() { private static ClientContext getClientContext() { Configuration conf = new YarnConfiguration(); ClientContext clientContext = new ClientContext(); - clientContext.setConfiguration(conf); + clientContext.setYarnConfig(conf); RuntimeFactory runtimeFactory = RuntimeFactory.getRuntimeFactory( clientContext); clientContext.setRuntimeFactory(runtimeFactory); diff --git a/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterManager.java b/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterManager.java index e303ccbaa0..851130ebd3 100644 --- a/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterManager.java +++ b/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterManager.java @@ -125,7 +125,7 @@ public abstract class ClusterManager { private static Logger LOG = LoggerFactory.getLogger(ClusterManager.class); - public final SubmarineConfiguration sconf = SubmarineConfiguration.create(); + public final SubmarineConfiguration sconf = SubmarineConfiguration.getInstance(); protected Collection clusterNodes = new ArrayList<>(); diff --git a/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterMonitor.java b/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterMonitor.java index ddc552728e..afbd3e69cd 100644 --- a/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterMonitor.java +++ b/submarine-commons/commons-cluster/src/main/java/org/apache/submarine/commons/cluster/ClusterMonitor.java @@ -75,7 +75,7 @@ public class ClusterMonitor { public ClusterMonitor(ClusterManager clusterManager) { this.clusterManager = clusterManager; - SubmarineConfiguration sconf = SubmarineConfiguration.create(); + SubmarineConfiguration sconf = SubmarineConfiguration.getInstance(); heartbeatInterval = sconf.getClusterHeartbeatInterval(); heartbeatTimeout = sconf.getClusterHeartbeatTimeout(); diff --git a/submarine-commons/commons-cluster/src/test/java/org/apache/submarine/commons/cluster/ClusterMultiNodeTest.java b/submarine-commons/commons-cluster/src/test/java/org/apache/submarine/commons/cluster/ClusterMultiNodeTest.java index 35af147c03..15aa24fde7 100644 --- a/submarine-commons/commons-cluster/src/test/java/org/apache/submarine/commons/cluster/ClusterMultiNodeTest.java +++ b/submarine-commons/commons-cluster/src/test/java/org/apache/submarine/commons/cluster/ClusterMultiNodeTest.java @@ -58,7 +58,7 @@ public static void startCluster() throws IOException, InterruptedException { } } LOG.info("clusterAddrList = {}", clusterAddrList); - SubmarineConfiguration sconf = SubmarineConfiguration.create(); + SubmarineConfiguration sconf = SubmarineConfiguration.getInstance(); sconf.setClusterAddress(clusterAddrList); // mock cluster manager server diff --git a/submarine-commons/commons-runtime/pom.xml b/submarine-commons/commons-runtime/pom.xml index 9e55f1f987..86b592302f 100644 --- a/submarine-commons/commons-runtime/pom.xml +++ b/submarine-commons/commons-runtime/pom.xml @@ -34,6 +34,17 @@ Submarine: Commons Runtime + + org.apache.submarine + commons-utils + ${project.version} + + + commons-logging + commons-logging + + + org.slf4j slf4j-api diff --git a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/ClientContext.java b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/ClientContext.java index 50f87fa933..a2a07c1e7f 100644 --- a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/ClientContext.java +++ b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/ClientContext.java @@ -19,7 +19,7 @@ package org.apache.submarine.commons.runtime; -import org.apache.submarine.commons.runtime.conf.SubmarineConfiguration; +import org.apache.submarine.commons.utils.SubmarineConfiguration; import org.apache.submarine.commons.runtime.fs.DefaultRemoteDirectoryManager; import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager; import org.apache.hadoop.conf.Configuration; @@ -31,11 +31,11 @@ public class ClientContext { private volatile RemoteDirectoryManager remoteDirectoryManager; private YarnClient yarnClient; - private Configuration submarineConfig; + private SubmarineConfiguration submarineConfig; private RuntimeFactory runtimeFactory; public ClientContext() { - submarineConfig = new SubmarineConfiguration(); + submarineConfig = SubmarineConfiguration.getInstance(); } public synchronized YarnClient getOrCreateYarnClient() { @@ -51,7 +51,7 @@ public Configuration getYarnConfig() { return yarnConf; } - public void setConfiguration(Configuration conf) { + public void setYarnConfig(Configuration conf) { this.yarnConf = conf; } @@ -66,11 +66,11 @@ public RemoteDirectoryManager getRemoteDirectoryManager() { return remoteDirectoryManager; } - public Configuration getSubmarineConfig() { + public SubmarineConfiguration getSubmarineConfig() { return submarineConfig; } - public void setSubmarineConfig(Configuration submarineConfig) { + public void setSubmarineConfig(SubmarineConfiguration submarineConfig) { this.submarineConfig = submarineConfig; } diff --git a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java index f7f1b81b65..b838a87923 100644 --- a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java +++ b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java @@ -20,8 +20,7 @@ package org.apache.submarine.commons.runtime; import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.conf.Configuration; -import org.apache.submarine.commons.runtime.conf.SubmarineConfiguration; +import org.apache.submarine.commons.utils.SubmarineConfiguration; import org.apache.submarine.commons.runtime.exception.SubmarineRuntimeException; import org.apache.submarine.commons.runtime.fs.SubmarineStorage; @@ -39,11 +38,10 @@ public RuntimeFactory(ClientContext clientContext) { public static RuntimeFactory getRuntimeFactory( ClientContext clientContext) { - Configuration submarineConfiguration = + SubmarineConfiguration submarineConfiguration = clientContext.getSubmarineConfig(); - String runtimeClass = submarineConfiguration.get( - SubmarineConfiguration.RUNTIME_CLASS, - SubmarineConfiguration.DEFAULT_RUNTIME_CLASS); + String runtimeClass = submarineConfiguration.getString( + SubmarineConfiguration.ConfVars.SUBMARINE_RUNTIME_CLASS); try { Class runtimeClazz = Class.forName(runtimeClass); diff --git a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java deleted file mode 100644 index b51446b07f..0000000000 --- a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/conf/SubmarineConfiguration.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.submarine.commons.runtime.conf; - -import org.apache.hadoop.conf.Configuration; - -public class SubmarineConfiguration extends Configuration { - private static final String SUBMARINE_CONFIGURATION_FILE = "submarine.xml"; - - public static final String SUBMARINE_CONFIGURATION_PREFIX = "submarine."; - - public static final String SUBMARINE_LOCALIZATION_PREFIX = - SUBMARINE_CONFIGURATION_PREFIX + "localization."; - /** - * Limit the size of directory/file to be localized. - * To avoid exhausting local disk space, - * this limit both remote and local file to be localized - */ - public static final String LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB = - SUBMARINE_LOCALIZATION_PREFIX + "max-allowed-file-size-mb"; - - // Default 2GB - public static final long DEFAULT_MAX_ALLOWED_REMOTE_URI_SIZE_MB = 2048; - - public static final String SUBMARINE_RUNTIME_APP_TYPE = "SUBMARINE"; - - public SubmarineConfiguration() { - this(new Configuration(false), true); - } - - public SubmarineConfiguration(Configuration configuration) { - this(configuration, false); - } - - public SubmarineConfiguration(Configuration configuration, - boolean loadLocalConfig) { - super(configuration); - if (loadLocalConfig) { - addResource(SUBMARINE_CONFIGURATION_FILE); - } - } - - private static final String PREFIX = "submarine."; - - public static final String RUNTIME_CLASS = PREFIX + "runtime.class"; - public static final String DEFAULT_RUNTIME_CLASS = - "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"; - - public void setSubmarineRuntimeClass(String runtimeClass) { - set(RUNTIME_CLASS, runtimeClass); - } -} diff --git a/submarine-commons/commons-utils/pom.xml b/submarine-commons/commons-utils/pom.xml index 9aecea422e..73396e79bd 100644 --- a/submarine-commons/commons-utils/pom.xml +++ b/submarine-commons/commons-utils/pom.xml @@ -39,18 +39,11 @@ commons-configuration commons-configuration ${commons-configuration.version} - - - commons-lang - commons-lang - - commons-lang commons-lang - ${commons-lang.version} diff --git a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java index 44cea260d1..f9dea58636 100644 --- a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java +++ b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java @@ -38,6 +38,8 @@ public class SubmarineConfiguration extends XMLConfiguration { private static final String SUBMARINE_SITE_XML = "submarine-site.xml"; + public static final String SUBMARINE_RUNTIME_APP_TYPE = "SUBMARINE"; + private static SubmarineConfiguration conf; private Map properties = new HashMap<>(); @@ -81,11 +83,19 @@ private SubmarineConfiguration() { } } - public static synchronized SubmarineConfiguration create() { - if (conf != null) { - return conf; + public static SubmarineConfiguration getInstance() { + if (conf == null) { + synchronized (SubmarineConfiguration.class) { + if (conf == null) { + conf = newInstance(); + } + } } + return conf; + } + public static SubmarineConfiguration newInstance() { + SubmarineConfiguration submarineConfig; ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); URL url; @@ -102,25 +112,18 @@ public static synchronized SubmarineConfiguration create() { if (url == null) { LOG.warn("Failed to load configuration, proceeding with a default"); - conf = new SubmarineConfiguration(); + submarineConfig = new SubmarineConfiguration(); } else { try { LOG.info("Load configuration from " + url); - conf = new SubmarineConfiguration(url); + submarineConfig = new SubmarineConfiguration(url); } catch (ConfigurationException e) { LOG.warn("Failed to load configuration from " + url + " proceeding with a default", e); - conf = new SubmarineConfiguration(); + submarineConfig = new SubmarineConfiguration(); } } - LOG.info("Workbench server Host: " + conf.getServerAddress()); - if (conf.useSsl() == false) { - LOG.info("Workbench server Port: " + conf.getServerPort()); - } else { - LOG.info("Workbench server SSL Port: " + conf.getServerSslPort()); - } - - return conf; + return submarineConfig; } public String getServerAddress() { @@ -351,10 +354,18 @@ public String getString(String envName, String propertyName, String defaultValue return getStringValue(propertyName, defaultValue); } + public void setString(ConfVars c, String value) { + properties.put(c.getVarName(), value); + } + public int getInt(ConfVars c) { return getInt(c.name(), c.getVarName(), c.getIntValue()); } + public void setInt(ConfVars c, int value) { + properties.put(c.getVarName(), String.valueOf(value)); + } + public int getInt(String envName, String propertyName, int defaultValue) { if (System.getenv(envName) != null) { return Integer.parseInt(System.getenv(envName)); @@ -370,6 +381,10 @@ public long getLong(ConfVars c) { return getLong(c.name(), c.getVarName(), c.getLongValue()); } + public void setLong(ConfVars c, long value) { + properties.put(c.getVarName(), String.valueOf(value)); + } + public long getLong(String envName, String propertyName, long defaultValue) { if (System.getenv(envName) != null) { return Long.parseLong(System.getenv(envName)); @@ -412,6 +427,11 @@ public boolean getBoolean(String envName, String propertyName, boolean defaultVa public enum ConfVars { SUBMARINE_CONF_DIR("submarine.conf.dir", "conf"), + SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB( + "submarine.localization.max-allowed-file-size-mb", 2048L), + SUBMARINE_RUNTIME_CLASS( + "submarine.runtime.class", + "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"), SERVER_ADDR("workbench.server.addr", "0.0.0.0"), SERVER_PORT("workbench.server.port", 8080), SERVER_SSL("workbench.server.ssl", false), diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java index 298081e31c..607d4f9dd5 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java @@ -45,7 +45,7 @@ public class JobServer { private static final Logger LOG = LoggerFactory.getLogger(JobServer.class); - private SubmarineConfiguration conf = SubmarineConfiguration.create(); + private SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); private Server jobServer; diff --git a/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java b/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java index 277c585d34..79b3da64a9 100644 --- a/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java +++ b/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java @@ -33,7 +33,7 @@ import java.util.List; import java.util.Map; -import static org.apache.submarine.commons.runtime.conf.SubmarineConfiguration.SUBMARINE_RUNTIME_APP_TYPE; +import static org.apache.submarine.commons.utils.SubmarineConfiguration.SUBMARINE_RUNTIME_APP_TYPE; /** * Utilities for YARN Runtime. diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java index 261286e1c5..219d6a7912 100644 --- a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java +++ b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java @@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.submarine.commons.runtime.ClientContext; -import org.apache.submarine.commons.runtime.conf.SubmarineConfiguration; +import org.apache.submarine.commons.utils.SubmarineConfiguration; import org.apache.submarine.commons.runtime.conf.SubmarineLogs; import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager; import org.apache.submarine.server.submitter.yarnservice.utils.ZipUtilities; @@ -54,7 +54,7 @@ public class FileSystemOperations { private static final Logger LOG = LoggerFactory.getLogger(FileSystemOperations.class); - private final Configuration submarineConfig; + private final SubmarineConfiguration submarineConfig; private final Configuration yarnConfig; private Set uploadedFiles = new HashSet<>(); @@ -179,9 +179,9 @@ public void validFileSize(String uri) throws IOException { } else { actualSizeByte = FileUtil.getDU(new File(uri)); } - long maxFileSizeMB = submarineConfig - .getLong(SubmarineConfiguration.LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, - SubmarineConfiguration.DEFAULT_MAX_ALLOWED_REMOTE_URI_SIZE_MB); + long maxFileSizeMB = submarineConfig.getLong( + SubmarineConfiguration.ConfVars. + SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB); LOG.info("{} fie/dir: {}, size(Byte):{}," + " Allowed max file/dir size: {}", locationType, uri, actualSizeByte, maxFileSizeMB * 1024 * 1024); diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliLocalization.java b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliLocalization.java index a6a773107a..ac38e1574b 100644 --- a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliLocalization.java +++ b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/TestYarnServiceRunJobCliLocalization.java @@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.submarine.client.cli.runjob.RunJobCli; import org.apache.submarine.commons.runtime.MockClientContext; -import org.apache.submarine.commons.runtime.conf.SubmarineConfiguration; +import org.apache.submarine.commons.utils.SubmarineConfiguration; import org.apache.submarine.commons.runtime.conf.SubmarineLogs; import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager; @@ -415,12 +415,13 @@ public void testRunJobRemoteUriExceedLocalizationSize() throws Exception { String localUri1 = "/temp/script2"; String containerLocal3 = "./"; - SubmarineConfiguration submarineConf = new SubmarineConfiguration(); + SubmarineConfiguration submarineConf = + SubmarineConfiguration.newInstance(); // Max 10MB, mock remote will always return file size 100MB. - submarineConf.set( - SubmarineConfiguration.LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, - "10"); + submarineConf.setLong( + SubmarineConfiguration.ConfVars. + SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 10L); mockClientContext.setSubmarineConfig(submarineConf); assertFalse(SubmarineLogs.isVerbose()); diff --git a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java b/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java index 202f6a2dcc..57897629c2 100644 --- a/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java +++ b/submarine-workbench/interpreter/interpreter-engine/src/main/java/org/apache/submarine/interpreter/InterpreterProcess.java @@ -58,7 +58,7 @@ public class InterpreterProcess extends Thread implements Interpreter { // cluster manager client private ClusterClient clusterClient = ClusterClient.getInstance(); - private SubmarineConfiguration sconf = SubmarineConfiguration.create(); + private SubmarineConfiguration sconf = SubmarineConfiguration.getInstance(); protected String interpreterId; diff --git a/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/InterpreterClusterTest.java b/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/InterpreterClusterTest.java index f7673c3310..2a2da4a8ea 100644 --- a/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/InterpreterClusterTest.java +++ b/submarine-workbench/interpreter/python-interpreter/src/test/java/org/apache/submarine/interpreter/InterpreterClusterTest.java @@ -52,7 +52,7 @@ public class InterpreterClusterTest { public static void startCluster() throws IOException, InterruptedException { LOG.info("startCluster >>>"); - sconf = SubmarineConfiguration.create(); + sconf = SubmarineConfiguration.getInstance(); // Set the cluster IP and port serverHost = NetworkUtils.findAvailableHostAddress(); diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java index 40ac9a8059..94cb76a503 100755 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java +++ b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java @@ -47,7 +47,7 @@ public class MyBatisUtil { checkCalledByTestMethod(); - SubmarineConfiguration conf = SubmarineConfiguration.create(); + SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); String jdbcClassName = conf.getJdbcDriverClassName(); String jdbcUrl = conf.getJdbcUrl(); String jdbcUserName = conf.getJdbcUserName(); @@ -95,7 +95,7 @@ private static void checkCalledByTestMethod() { private static void usingTestDatabase() { LOG.info("Run the test unit using the test database"); // Run the test unit using the test database - SubmarineConfiguration conf = SubmarineConfiguration.create(); + SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); conf.setJdbcUrl("jdbc:mysql://127.0.0.1:3306/submarineDB_test?" + "useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&" + "failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false"); diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java index 998d835eb0..ad1dc749bc 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java +++ b/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java @@ -59,12 +59,18 @@ public class WorkbenchServer extends ResourceConfig { public static Server jettyWebServer; public static ServiceLocator sharedServiceLocator; - private static SubmarineConfiguration conf = SubmarineConfiguration.create(); + private static SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); public static void main(String[] args) throws InterruptedException { PropertyConfigurator.configure(ClassLoader.getSystemResource("log4j.properties")); - final SubmarineConfiguration conf = SubmarineConfiguration.create(); + final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); + LOG.info("Workbench server Host: " + conf.getServerAddress()); + if (conf.useSsl() == false) { + LOG.info("Workbench server Port: " + conf.getServerPort()); + } else { + LOG.info("Workbench server SSL Port: " + conf.getServerSslPort()); + } jettyWebServer = setupJettyServer(conf); diff --git a/submarine-workbench/workbench-server/src/test/java/org/apache/submarine/server/WorkbenchClusterServerTest.java b/submarine-workbench/workbench-server/src/test/java/org/apache/submarine/server/WorkbenchClusterServerTest.java index 8ae8d91e24..9c1f863afa 100644 --- a/submarine-workbench/workbench-server/src/test/java/org/apache/submarine/server/WorkbenchClusterServerTest.java +++ b/submarine-workbench/workbench-server/src/test/java/org/apache/submarine/server/WorkbenchClusterServerTest.java @@ -45,7 +45,7 @@ public class WorkbenchClusterServerTest { public static void start() throws Exception { LOG.info("WorkbenchClusterServerTest:start()"); - SubmarineConfiguration conf = SubmarineConfiguration.create(); + SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); String serverHost = NetworkUtils.findAvailableHostAddress(); int serverPort = NetworkUtils.findRandomAvailablePortOnAllLocalInterfaces(); String clusterAdd = serverHost + ":" + serverPort; From aae0380151f79c7e93f8a05ab1b979d3c4c5ef9b Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Mon, 18 Nov 2019 11:05:31 +0100 Subject: [PATCH 07/23] SUBMARINE-66. Improve TF config env JSON generator + tests ### What is this PR for? * The goal is to update the TensorFlow config generation. Also some of the tests were doing some manual JSON thing which is not very maintainable - it is also changed to a more flexible one using jackson-databind ### What type of PR is it? * Refactoring | Test ### Todos * [x] - Travis checks * [x] - New tests should pass ### What is the Jira issue? * [SUBMARINE-66](https://issues.apache.org/jira/browse/SUBMARINE-66) ### How should this be tested? * Only the new UTs should pass. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? Yes * We should probably add jackson-module-jaxb-annotations? * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Adam Antal Closes #91 from adamantal/SUBMARINE-66 and squashes the following commits: 268e7e4 [Adam Antal] Renaming setUp to setup in TensorFlowConfigEnvGeneratorTest ed3b220 [Adam Antal] SUBMARINE-66. Improve TF config env JSON generator + tests --- pom.xml | 1 + submarine-all/pom.xml | 22 +++ .../server-submitter/submitter-yarn/pom.xml | 4 + .../submitter-yarnservice/pom.xml | 43 +++++ .../tensorflow/TensorFlowCommons.java | 55 ------- .../TensorFlowConfigEnvGenerator.java | 102 ++++++++++++ .../command/TensorFlowLaunchCommand.java | 3 +- .../yarnservice/TestTFConfigGenerator.java | 73 --------- .../TensorFlowConfigEnvGeneratorTest.java | 149 ++++++++++++++++++ 9 files changed, 323 insertions(+), 129 deletions(-) create mode 100644 submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowConfigEnvGenerator.java delete mode 100644 submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/TestTFConfigGenerator.java create mode 100644 submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowConfigEnvGeneratorTest.java diff --git a/pom.xml b/pom.xml index 82234c770e..5465cd7b46 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ 2.8.1 2.9.10 2.9.10 + 2.9.10 1.10 3.1 diff --git a/submarine-all/pom.xml b/submarine-all/pom.xml index 952709a2b5..900a5448a9 100644 --- a/submarine-all/pom.xml +++ b/submarine-all/pom.xml @@ -95,6 +95,12 @@ org.apache.hadoop hadoop-hdfs-client ${hadoop.version} + + + com.fasterxml.jackson.core + jackson-databind + + org.apache.hadoop @@ -105,6 +111,10 @@ io.netty netty + + com.fasterxml.jackson.core + jackson-databind + @@ -128,6 +138,12 @@ org.apache.hadoop hadoop-hdfs-client ${hadoop.version} + + + com.fasterxml.jackson.core + jackson-databind + + @@ -142,6 +158,12 @@ org.apache.hadoop hadoop-hdfs-client ${hadoop.version} + + + com.fasterxml.jackson.core + jackson-databind + + diff --git a/submarine-server/server-submitter/submitter-yarn/pom.xml b/submarine-server/server-submitter/submitter-yarn/pom.xml index 77da805864..31d0040fa9 100644 --- a/submarine-server/server-submitter/submitter-yarn/pom.xml +++ b/submarine-server/server-submitter/submitter-yarn/pom.xml @@ -171,6 +171,10 @@ commons-codec commons-codec + + com.fasterxml.jackson.core + jackson-databind + diff --git a/submarine-server/server-submitter/submitter-yarnservice/pom.xml b/submarine-server/server-submitter/submitter-yarnservice/pom.xml index 88b862599e..60e0217665 100644 --- a/submarine-server/server-submitter/submitter-yarnservice/pom.xml +++ b/submarine-server/server-submitter/submitter-yarnservice/pom.xml @@ -104,6 +104,10 @@ commons-io commons-io + + com.fasterxml.jackson.core + jackson-databind + @@ -142,6 +146,14 @@ org.codehaus.jackson jackson-mapper-asl + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + @@ -225,6 +237,10 @@ org.codehaus.jackson jackson-mapper-asl + + com.fasterxml.jackson.core + jackson-databind + @@ -256,6 +272,10 @@ org.codehaus.jackson jackson-mapper-asl + + com.fasterxml.jackson.core + jackson-databind + @@ -273,8 +293,31 @@ commons-io commons-io + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson-annotations.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + ${jackson-module-jaxb-annotations.version} + diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowCommons.java b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowCommons.java index 424fc6dc2d..32d2640654 100644 --- a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowCommons.java +++ b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowCommons.java @@ -24,7 +24,6 @@ import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.submarine.commons.runtime.conf.Envs; import org.apache.submarine.commons.runtime.api.Role; -import org.apache.submarine.server.submitter.yarnservice.YarnServiceUtils; import java.util.Map; @@ -55,58 +54,4 @@ public static String getDNSDomain(Configuration yarnConfig) { public static String getScriptFileName(Role role) { return "run-" + role.getName() + ".sh"; } - - public static String getTFConfigEnv(String componentName, int nWorkers, - int nPs, String serviceName, String userName, String domain) { - String commonEndpointSuffix = YarnServiceUtils - .getDNSNameCommonSuffix(serviceName, userName, domain, 8000); - - String json = "{\\\"cluster\\\":{"; - - String master = getComponentArrayJson("master", 1, commonEndpointSuffix) - + ","; - String worker = getComponentArrayJson("worker", nWorkers - 1, - commonEndpointSuffix) + ","; - String ps = getComponentArrayJson("ps", nPs, commonEndpointSuffix) + "},"; - - StringBuilder sb = new StringBuilder(); - sb.append("\\\"task\\\":{"); - sb.append(" \\\"type\\\":\\\""); - sb.append(componentName); - sb.append("\\\","); - sb.append(" \\\"index\\\":"); - sb.append('$'); - sb.append(Envs.TASK_INDEX_ENV + "},"); - String task = sb.toString(); - String environment = "\\\"environment\\\":\\\"cloud\\\"}"; - - sb = new StringBuilder(); - sb.append(json); - sb.append(master); - sb.append(worker); - sb.append(ps); - sb.append(task); - sb.append(environment); - return sb.toString(); - } - - private static String getComponentArrayJson(String componentName, int count, - String endpointSuffix) { - String component = "\\\"" + componentName + "\\\":"; - StringBuilder array = new StringBuilder(); - array.append("["); - for (int i = 0; i < count; i++) { - array.append("\\\""); - array.append(componentName); - array.append("-"); - array.append(i); - array.append(endpointSuffix); - array.append("\\\""); - if (i != count - 1) { - array.append(","); - } - } - array.append("]"); - return component + array.toString(); - } } diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowConfigEnvGenerator.java b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowConfigEnvGenerator.java new file mode 100644 index 0000000000..45de9e71c7 --- /dev/null +++ b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowConfigEnvGenerator.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.yarnservice.tensorflow; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; +import org.apache.submarine.commons.runtime.conf.Envs; +import org.apache.submarine.server.submitter.yarnservice.YarnServiceUtils; + +public class TensorFlowConfigEnvGenerator { + + private static final ObjectMapper OBJECT_MAPPER = createObjectMapper(); + + private static ObjectMapper createObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.setAnnotationIntrospector( + new JaxbAnnotationIntrospector(TypeFactory.defaultInstance())); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + mapper.configure(SerializationFeature.FLUSH_AFTER_WRITE_VALUE, false); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return mapper; + } + + public static String getTFConfigEnv(String componentName, int nWorkers, + int nPs, String serviceName, String userName, String domain) { + String commonEndpointSuffix = YarnServiceUtils + .getDNSNameCommonSuffix(serviceName, userName, domain, 8000); + + TFConfigEnv tfConfigEnv = + new TFConfigEnv(nWorkers, nPs, componentName, commonEndpointSuffix); + return tfConfigEnv.toJson(); + } + + private static class TFConfigEnv { + private final int nWorkers; + private final int nPS; + private final String componentName; + private final String endpointSuffix; + + TFConfigEnv(int nWorkers, int nPS, String componentName, + String endpointSuffix) { + this.nWorkers = nWorkers; + this.nPS = nPS; + this.componentName = componentName; + this.endpointSuffix = endpointSuffix; + } + + String toJson() { + ObjectNode rootNode = OBJECT_MAPPER.createObjectNode(); + + ObjectNode cluster = rootNode.putObject("cluster"); + createComponentArray(cluster, "master", 1); + createComponentArray(cluster, "worker", nWorkers - 1); + createComponentArray(cluster, "ps", nPS); + + ObjectNode task = rootNode.putObject("task"); + task.put("type", componentName); + task.put("index", "$" + Envs.TASK_INDEX_ENV); + task.put("environment", "cloud"); + try { + return OBJECT_MAPPER.writeValueAsString(rootNode); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize TF config env JSON!", + e); + } + } + + private void createComponentArray(ObjectNode cluster, String name, + int count) { + ArrayNode array = cluster.putArray(name); + for (int i = 0; i < count; i++) { + String componentValue = String.format("%s-%d%s", name, i, + endpointSuffix); + array.add(componentValue); + } + } + } +} diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorFlowLaunchCommand.java b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorFlowLaunchCommand.java index b88ea2481c..13c83cc849 100644 --- a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorFlowLaunchCommand.java +++ b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorFlowLaunchCommand.java @@ -27,6 +27,7 @@ import org.apache.submarine.server.submitter.yarnservice.command.AbstractLaunchCommand; import org.apache.submarine.server.submitter.yarnservice.command.LaunchScriptBuilder; import org.apache.submarine.server.submitter.yarnservice.tensorflow.TensorFlowCommons; +import org.apache.submarine.server.submitter.yarnservice.tensorflow.TensorFlowConfigEnvGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +79,7 @@ public String generateLaunchScript() throws IOException { // When distributed training is required if (distributed) { - String tfConfigEnvValue = TensorFlowCommons.getTFConfigEnv( + String tfConfigEnvValue = TensorFlowConfigEnvGenerator.getTFConfigEnv( role.getComponentName(), numberOfWorkers, numberOfPS, name, TensorFlowCommons.getUserName(), diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/TestTFConfigGenerator.java b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/TestTFConfigGenerator.java deleted file mode 100644 index 072eaec569..0000000000 --- a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/TestTFConfigGenerator.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.submarine.server.submitter.yarnservice; - -import org.apache.submarine.server.submitter.yarnservice.tensorflow.TensorFlowCommons; -import org.codehaus.jettison.json.JSONException; -import org.junit.Assert; -import org.junit.Test; - -/** - * Class to test some functionality of {@link TensorFlowCommons}. - */ -public class TestTFConfigGenerator { - @Test - public void testSimpleDistributedTFConfigGenerator() throws JSONException { - String json = TensorFlowCommons.getTFConfigEnv("worker", 5, 3, "wtan", - "tf-job-001", "example.com"); - String expected = - "{\\\"cluster\\\":{\\\"master\\\":[\\\"master-0.wtan.tf-job-001.example.com:8000\\\"]," + - "\\\"worker\\\":[\\\"worker-0.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"worker-1.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"worker-2.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"worker-3.wtan.tf-job-001.example.com:8000\\\"]," + - "\\\"ps\\\":[\\\"ps-0.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"ps-1.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"ps-2.wtan.tf-job-001.example.com:8000\\\"]}," + - "\\\"task\\\":{ \\\"type\\\":\\\"worker\\\", \\\"index\\\":$_TASK_INDEX}," + - "\\\"environment\\\":\\\"cloud\\\"}"; - Assert.assertEquals(expected, json); - - json = TensorFlowCommons.getTFConfigEnv("ps", 5, 3, "wtan", "tf-job-001", - "example.com"); - expected = - "{\\\"cluster\\\":{\\\"master\\\":[\\\"master-0.wtan.tf-job-001.example.com:8000\\\"]," + - "\\\"worker\\\":[\\\"worker-0.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"worker-1.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"worker-2.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"worker-3.wtan.tf-job-001.example.com:8000\\\"]," + - "\\\"ps\\\":[\\\"ps-0.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"ps-1.wtan.tf-job-001.example.com:8000\\\"," + - "\\\"ps-2.wtan.tf-job-001.example.com:8000\\\"]}," + - "\\\"task\\\":{ \\\"type\\\":\\\"ps\\\", \\\"index\\\":$_TASK_INDEX}," + - "\\\"environment\\\":\\\"cloud\\\"}"; - Assert.assertEquals(expected, json); - - json = TensorFlowCommons.getTFConfigEnv("master", 2, 1, "wtan", "tf-job-001", - "example.com"); - expected = - "{\\\"cluster\\\":{\\\"master\\\":[\\\"master-0.wtan.tf-job-001.example.com:8000\\\"]," + - "\\\"worker\\\":[\\\"worker-0.wtan.tf-job-001.example.com:8000\\\"]," + - "\\\"ps\\\":[\\\"ps-0.wtan.tf-job-001.example.com:8000\\\"]}," + - "\\\"task\\\":{ \\\"type\\\":\\\"master\\\", \\\"index\\\":$_TASK_INDEX}," + - "\\\"environment\\\":\\\"cloud\\\"}"; - Assert.assertEquals(expected, json); - } -} diff --git a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowConfigEnvGeneratorTest.java b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowConfigEnvGeneratorTest.java new file mode 100644 index 0000000000..de1135b4aa --- /dev/null +++ b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/TensorFlowConfigEnvGeneratorTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.submarine.server.submitter.yarnservice.tensorflow; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeType; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Class to test some functionality of {@link TensorFlowConfigEnvGenerator}. + */ +public class TensorFlowConfigEnvGeneratorTest { + private ObjectMapper objectMapper; + + @Before + public void setup() { + objectMapper = new ObjectMapper(); + } + + private void verifyCommonJsonData(JsonNode node, String taskType) { + JsonNode task = node.get("task"); + assertNotNull(task); + assertEquals(taskType, task.get("type").asText()); + assertEquals("$_TASK_INDEX", task.get("index").asText()); + + JsonNode environment = task.get("environment"); + assertNotNull(environment); + assertEquals("cloud", environment.asText()); + } + + private void verifyArrayElements(JsonNode node, String childName, + String... elements) { + JsonNode master = node.get(childName); + assertNotNull(master); + assertEquals(JsonNodeType.ARRAY, master.getNodeType()); + ArrayNode masterArray = (ArrayNode) master; + verifyArray(masterArray, elements); + } + + private void verifyArray(ArrayNode array, String... elements) { + int arraySize = array.size(); + assertEquals(elements.length, arraySize); + + for (int i = 0; i < arraySize; i++) { + JsonNode arrayElement = array.get(i); + assertEquals(elements[i], arrayElement.asText()); + } + } + + @Test + public void testSimpleDistributedTFConfigGeneratorWorker() + throws IOException { + String json = TensorFlowConfigEnvGenerator.getTFConfigEnv("worker", 5, 3, + "wtan", "tf-job-001", "example.com"); + + JsonNode jsonNode = objectMapper.readTree(json); + assertNotNull(jsonNode); + JsonNode cluster = jsonNode.get("cluster"); + assertNotNull(cluster); + + verifyArrayElements(cluster, "master", + "master-0.wtan.tf-job-001.example.com:8000"); + verifyArrayElements(cluster, "worker", + "worker-0.wtan.tf-job-001.example.com:8000", + "worker-1.wtan.tf-job-001.example.com:8000", + "worker-2.wtan.tf-job-001.example.com:8000", + "worker-3.wtan.tf-job-001.example.com:8000"); + + verifyArrayElements(cluster, "ps", + "ps-0.wtan.tf-job-001.example.com:8000", + "ps-1.wtan.tf-job-001.example.com:8000", + "ps-2.wtan.tf-job-001.example.com:8000"); + + verifyCommonJsonData(jsonNode, "worker"); + } + + @Test + public void testSimpleDistributedTFConfigGeneratorMaster() + throws IOException { + String json = TensorFlowConfigEnvGenerator.getTFConfigEnv("master", 2, 1, + "wtan", "tf-job-001", "example.com"); + + JsonNode jsonNode = objectMapper.readTree(json); + assertNotNull(jsonNode); + JsonNode cluster = jsonNode.get("cluster"); + assertNotNull(cluster); + + verifyArrayElements(cluster, "master", + "master-0.wtan.tf-job-001.example.com:8000"); + verifyArrayElements(cluster, "worker", + "worker-0.wtan.tf-job-001.example.com:8000"); + + verifyArrayElements(cluster, "ps", + "ps-0.wtan.tf-job-001.example.com:8000"); + + verifyCommonJsonData(jsonNode, "master"); + } + + @Test + public void testSimpleDistributedTFConfigGeneratorPS() throws IOException { + String json = TensorFlowConfigEnvGenerator.getTFConfigEnv("ps", 5, 3, + "wtan", "tf-job-001", "example.com"); + + JsonNode jsonNode = objectMapper.readTree(json); + assertNotNull(jsonNode); + JsonNode cluster = jsonNode.get("cluster"); + assertNotNull(cluster); + + verifyArrayElements(cluster, "master", + "master-0.wtan.tf-job-001.example.com:8000"); + verifyArrayElements(cluster, "worker", + "worker-0.wtan.tf-job-001.example.com:8000", + "worker-1.wtan.tf-job-001.example.com:8000", + "worker-2.wtan.tf-job-001.example.com:8000", + "worker-3.wtan.tf-job-001.example.com:8000"); + + verifyArrayElements(cluster, "ps", + "ps-0.wtan.tf-job-001.example.com:8000", + "ps-1.wtan.tf-job-001.example.com:8000", + "ps-2.wtan.tf-job-001.example.com:8000"); + + verifyCommonJsonData(jsonNode, "ps"); + } +} From 51192a8f50d99249fdf3847b8f766e2effa0dcc7 Mon Sep 17 00:00:00 2001 From: Xun Liu Date: Mon, 18 Nov 2019 13:11:11 +0800 Subject: [PATCH 08/23] SUBMARINE-292. Merge workbench server into submarine server ### What is this PR for? Now submarine has 2 services, submarine server and workbench server. These two services have no conflicts. Originally divided into two services, it is for the development of sub-modules, easy to develop. But this way 1. Users need to maintain two service processes, which increases the cost of use and maintenance of users and administrators. 2. Because we need to be able to run the service on multiple resource scheduling systems(YARN/K8s/Docker), the two services create the complexity of the launcher module. So, now the service in the workbench server is migrated to the submarine server. But the code for the workbench server remains the same. ### What type of PR is it? Refactoring ### Todos * [ ] [SUBMARINE-293](https://issues.apache.org/jira/browse/SUBMARINE-293) - Add submarine server rest test case ### What is the Jira issue? * https://issues.apache.org/jira/browse/SUBMARINE-292 ### How should this be tested? * [CI Pass](https://travis-ci.org/liuxunorg/submarine/builds/613280558) ### Screenshots (if appropriate) ``` |____bin | |____common.sh | |____submarine-daemon.sh | |____submarine-env.sh |____commons-runtime-0.3.0-SNAPSHOT.jar |____conf | |____log4j.properties | |____log4j.properties.template | |____submarine-env.sh.template | |____submarine-site.xml | |____submarine-site.xml.template |____lib | |____bcpg-jdk15on-1.61.jar | |____bcpkix-jdk15on-1.61.jar | |____hk2-utils-2.5.0-b42.jar | |____interpreter | | |____python | | | |____python-interpreter-0.3.0-SNAPSHOT-shade.jar | | |____spark | | | |____scala-2.11 | | | | |____spark-scala-2.11-0.9.0-SNAPSHOT.jar | | | |____spark-interpreter-0.3.0-SNAPSHOT-shade.jar | |____j2objc-annotations-1.1.jar | |____jackson-annotations-2.9.10.jar | |____websocket-server-9.4.18.v20190429.jar | |____workbench | | |____workbench-web.war |____LICENSE |____licenses | |____LICENSE-asm.txt | |____LICENSE-hamcrest.txt | |____LICENSE-slf4j.txt | |____LICENSE-tony.txt |____logs | |____submarine.log |____NOTICE |____server |____submarine-all-0.3.0-SNAPSHOT-hadoop-2.9.jar |____submarine-client-0.3.0-SNAPSHOT.jar |____submitter-yarn-0.3.0-SNAPSHOT.jar |____tony-core-0.3.21.jar ``` ### Questions: * Does the licenses files need update? Yes/No * Is there breaking changes for older versions? Yes/No * Does this needs documentation? Yes/No Author: Xun Liu Closes #93 from liuxunorg/SUBMARINE-292 and squashes the following commits: aa5508b [Xun Liu] Fixed some issue. 47f30da [Xun Liu] SUBMARINE-292. Merge workbench server into submarine server --- .travis.yml | 9 +- bin/common.sh | 14 +- ...orkbench-daemon.sh => submarine-daemon.sh} | 65 ++-- bin/submarine-env.sh | 22 ++ conf/submarine-env.sh.template | 4 +- conf/submarine-site.xml | 38 +- conf/submarine-site.xml.template | 38 +- docs/development/README.md | 2 +- .../commons/cluster/ClusterClient.java | 4 +- .../commons/cluster/ClusterManager.java | 4 +- .../commons/cluster/ClusterServer.java | 4 +- .../commons/utils/SubmarineConfiguration.java | 106 +++--- submarine-dist/src/assembly/distribution.xml | 31 +- submarine-server/pom.xml | 12 - submarine-server/server-core/pom.xml | 339 ++++++++++++++---- .../apache/submarine/jobserver/JobServer.java | 165 --------- .../submarine/server/SubmarineServer.java | 30 +- .../jobserver/rest/api/JobApi.java | 8 +- .../jobserver/rest/dao/Component.java | 2 +- .../jobserver/rest/dao/EnvVaraible.java | 2 +- .../rest/dao/JsonExclusionStrategy.java | 2 +- .../jobserver/rest/dao/JsonResponse.java | 14 +- .../jobserver/rest/dao/MLJobSpec.java | 2 +- .../jobserver/rest/dao/RestConstants.java | 14 +- .../rest/provider/YamlEntityProvider.java | 2 +- .../server/workbench}/annotation/Dict.java | 2 +- .../workbench}/annotation/SubmarineApi.java | 2 +- .../workbench}/database/MyBatisUtil.java | 2 +- .../database/entity/BaseEntity.java | 4 +- .../workbench}/database/entity/Project.java | 4 +- .../database/entity/ProjectFiles.java | 2 +- .../workbench}/database/entity/SysDept.java | 2 +- .../database/entity/SysDeptSelect.java | 2 +- .../database/entity/SysDeptTree.java | 2 +- .../workbench}/database/entity/SysDict.java | 2 +- .../database/entity/SysDictItem.java | 2 +- .../database/entity/SysMessage.java | 2 +- .../workbench}/database/entity/SysUser.java | 6 +- .../workbench}/database/entity/Team.java | 2 +- .../database/entity/TeamMember.java | 2 +- .../database/mappers/ProjectFilesMapper.java | 4 +- .../database/mappers/ProjectMapper.java | 4 +- .../database/mappers/SysDeptMapper.java | 4 +- .../database/mappers/SysDictItemMapper.java | 4 +- .../database/mappers/SysDictMapper.java | 4 +- .../database/mappers/SysMessageMapper.java | 4 +- .../database/mappers/SysUserMapper.java | 4 +- .../database/mappers/SystemMapper.java | 2 +- .../database/mappers/TeamMapper.java | 4 +- .../database/mappers/TeamMemberMapper.java | 4 +- .../database/service/ProjectFilesService.java | 8 +- .../database/service/ProjectService.java | 12 +- .../database/service/SysDictItemService.java | 10 +- .../database/service/SysMessageService.java | 8 +- .../database/service/SysUserService.java | 8 +- .../database/service/TeamMemberService.java | 8 +- .../database/service/TeamService.java | 12 +- .../utils/CustomJsonDateDeserializer.java | 2 +- .../database/utils/DepartmentUtil.java | 8 +- .../database/utils/DictAnnotation.java | 10 +- .../workbench}/database/utils/GitUtils.java | 2 +- .../database/utils/HttpRequestUtil.java | 2 +- .../database/utils/MybatisGeneratorMain.java | 4 +- .../server/workbench}/entity/Action.java | 2 +- .../server/workbench}/entity/Permission.java | 2 +- .../server/workbench}/entity/Role.java | 2 +- .../server/workbench}/entity/User.java | 2 +- .../server/workbench}/entity/UserInfo.java | 2 +- .../server/workbench}/rest/LoginRestApi.java | 12 +- .../workbench}/rest/ProjectRestApi.java | 12 +- .../workbench}/rest/SysDeptRestApi.java | 20 +- .../workbench}/rest/SysDictItemRestApi.java | 16 +- .../workbench}/rest/SysDictRestApi.java | 14 +- .../workbench}/rest/SysUserRestApi.java | 22 +- .../server/workbench}/rest/SystemRestApi.java | 16 +- .../server/workbench}/rest/TeamRestApi.java | 12 +- .../server/JsonExclusionStrategy.java | 2 +- .../workbench}/server/JsonResponse.java | 4 +- .../server/workbench}/utils/TestUtils.java | 6 +- .../websocket/ConnectionManager.java | 2 +- .../websocket/DateJsonDeserializer.java | 2 +- .../server/workbench}/websocket/Message.java | 2 +- .../workbench}/websocket/NotebookServer.java | 2 +- .../workbench}/websocket/NotebookSocket.java | 2 +- .../websocket/NotebookSocketListener.java | 2 +- .../websocket/NotebookWebSocketCreator.java | 2 +- .../src/main/resources/mbgConfiguration.xml | 0 .../src/main/resources/mybatis-config.xml | 0 .../database/mappers/ProjectFilesMapper.xml | 24 +- .../database/mappers/ProjectMapper.xml | 14 +- .../database/mappers/SysDeptMapper.xml | 10 +- .../database/mappers/SysDictItemMapper.xml | 10 +- .../database/mappers/SysDictMapper.xml | 10 +- .../database/mappers/SysMessageMapper.xml | 26 +- .../database/mappers/SysUserMapper.xml | 12 +- .../database/mappers/SystemMapper.xml | 2 +- .../submarine/database/mappers/TeamMapper.xml | 14 +- .../database/mappers/TeamMemberMapper.xml | 16 +- .../jobserver/rest/api}/JobApiTest.java | 47 ++- .../database/service/ProjectServiceTest.java | 6 +- .../database/service/SysUserServiceTest.java | 4 +- .../service/TeamMemberServiceTest.java | 4 +- .../database/service/TeamServiceTest.java | 6 +- .../database/utils/DepartmentUtilTest.java | 8 +- .../database/utils/DictAnnotationTest.java | 8 +- .../workbench}/rest/CommonDataTest.java | 18 +- .../workbench}/rest/SysDeptRestApiTest.java | 12 +- .../rest/SysDictItemRestApiTest.java | 10 +- .../workbench}/rest/SysDictRestApiTest.java | 8 +- .../workbench}/rest/SysUserRestApiTest.java | 6 +- .../server/AbstractWorkbenchServerTest.java | 9 +- .../workbench}/server/GitUtilsTest.java | 6 +- .../workbench}/server/JsonResponseTest.java | 6 +- .../server/WorkbenchClusterServerTest.java | 2 +- .../websocket/NotebookServerTest.java | 4 +- submarine-server/server-submitter/pom.xml | 2 +- .../server-submitter/submitter-k8s/pom.xml | 2 +- .../server-submitter/submitter-yarn/pom.xml | 2 +- .../submitter-yarnservice/pom.xml | 2 +- .../TestYarnServiceRunJobCliLocalization.java | 5 + .../interpreter/InterpreterProcess.java | 2 +- submarine-workbench/pom.xml | 1 - submarine-workbench/workbench-server/pom.xml | 279 -------------- .../src/main/resources/log4j.properties | 17 - 124 files changed, 782 insertions(+), 1095 deletions(-) rename bin/{workbench-daemon.sh => submarine-daemon.sh} (67%) create mode 100644 bin/submarine-env.sh delete mode 100644 submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java rename submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java => submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java (90%) rename submarine-server/server-core/src/main/java/org/apache/submarine/{ => server}/jobserver/rest/api/JobApi.java (91%) rename submarine-server/server-core/src/main/java/org/apache/submarine/{ => server}/jobserver/rest/dao/Component.java (96%) rename submarine-server/server-core/src/main/java/org/apache/submarine/{ => server}/jobserver/rest/dao/EnvVaraible.java (95%) rename submarine-server/server-core/src/main/java/org/apache/submarine/{ => server}/jobserver/rest/dao/JsonExclusionStrategy.java (95%) rename submarine-server/server-core/src/main/java/org/apache/submarine/{ => server}/jobserver/rest/dao/JsonResponse.java (93%) rename submarine-server/server-core/src/main/java/org/apache/submarine/{ => server}/jobserver/rest/dao/MLJobSpec.java (98%) rename submarine-server/server-core/src/main/java/org/apache/submarine/{ => server}/jobserver/rest/dao/RestConstants.java (70%) rename submarine-server/server-core/src/main/java/org/apache/submarine/{ => server}/jobserver/rest/provider/YamlEntityProvider.java (97%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/annotation/Dict.java (94%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/annotation/SubmarineApi.java (95%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/MyBatisUtil.java (98%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/BaseEntity.java (95%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/Project.java (96%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/ProjectFiles.java (95%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/SysDept.java (97%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/SysDeptSelect.java (97%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/SysDeptTree.java (96%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/SysDict.java (96%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/SysDictItem.java (96%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/SysMessage.java (96%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/SysUser.java (93%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/Team.java (96%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/entity/TeamMember.java (95%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/ProjectFilesMapper.java (90%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/ProjectMapper.java (90%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/SysDeptMapper.java (89%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/SysDictItemMapper.java (89%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/SysDictMapper.java (89%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/SysMessageMapper.java (89%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/SysUserMapper.java (90%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/SystemMapper.java (93%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/TeamMapper.java (90%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/mappers/TeamMemberMapper.java (90%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/service/ProjectFilesService.java (85%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/service/ProjectService.java (93%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/service/SysDictItemService.java (81%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/service/SysMessageService.java (83%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/service/SysUserService.java (94%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/service/TeamMemberService.java (90%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/service/TeamService.java (93%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/utils/CustomJsonDateDeserializer.java (96%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/utils/DepartmentUtil.java (93%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/utils/DictAnnotation.java (94%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/utils/GitUtils.java (99%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/utils/HttpRequestUtil.java (98%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/database/utils/MybatisGeneratorMain.java (95%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/entity/Action.java (96%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/entity/Permission.java (98%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/entity/Role.java (98%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/entity/User.java (98%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/entity/UserInfo.java (98%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/rest/LoginRestApi.java (86%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/rest/ProjectRestApi.java (91%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/rest/SysDeptRestApi.java (92%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/rest/SysDictItemRestApi.java (93%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/rest/SysDictRestApi.java (93%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/rest/SysUserRestApi.java (92%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/rest/SystemRestApi.java (88%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/rest/TeamRestApi.java (92%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/server/JsonExclusionStrategy.java (95%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/server/JsonResponse.java (97%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/utils/TestUtils.java (91%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/websocket/ConnectionManager.java (98%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/websocket/DateJsonDeserializer.java (97%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/websocket/Message.java (97%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/websocket/NotebookServer.java (98%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/websocket/NotebookSocket.java (97%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/websocket/NotebookSocketListener.java (94%) rename {submarine-workbench/workbench-server/src/main/java/org/apache/submarine => submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench}/websocket/NotebookWebSocketCreator.java (96%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/mbgConfiguration.xml (100%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/mybatis-config.xml (100%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/ProjectFilesMapper.xml (88%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/ProjectMapper.xml (92%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/SysDeptMapper.xml (90%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/SysDictItemMapper.xml (89%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/SysDictMapper.xml (89%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/SysMessageMapper.xml (89%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/SysUserMapper.xml (91%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/SystemMapper.xml (93%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/TeamMapper.xml (90%) rename {submarine-workbench/workbench-server => submarine-server/server-core}/src/main/resources/org/apache/submarine/database/mappers/TeamMemberMapper.xml (91%) rename submarine-server/server-core/src/test/java/{ => org/apache/submarine/jobserver/rest/api}/JobApiTest.java (80%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/database/service/ProjectServiceTest.java (97%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/database/service/SysUserServiceTest.java (98%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/database/service/TeamMemberServiceTest.java (94%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/database/service/TeamServiceTest.java (96%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/database/utils/DepartmentUtilTest.java (91%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/database/utils/DictAnnotationTest.java (89%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/rest/CommonDataTest.java (94%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/rest/SysDeptRestApiTest.java (94%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/rest/SysDictItemRestApiTest.java (95%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/rest/SysDictRestApiTest.java (94%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/rest/SysUserRestApiTest.java (94%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/server/AbstractWorkbenchServerTest.java (97%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/server/GitUtilsTest.java (97%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/server/JsonResponseTest.java (94%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/server/WorkbenchClusterServerTest.java (98%) rename {submarine-workbench/workbench-server/src/test/java/org/apache/submarine => submarine-server/server-core/src/test/java/org/apache/submarine/server/workbench}/websocket/NotebookServerTest.java (95%) delete mode 100644 submarine-workbench/workbench-server/pom.xml delete mode 100644 submarine-workbench/workbench-server/src/main/resources/log4j.properties diff --git a/.travis.yml b/.travis.yml index af2a24e01b..6e7daa1d32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,10 +40,9 @@ services: env: global: - # submarine core does not required by workbench-server integration tests # If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE - EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-submitter/submitter-k8s,!submarine-server/server-core" - - EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web,!submarine-workbench/workbench-server" + - EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web" - EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter"" - EXCLUDE_SUBMODULE_TONY="!submodules/tony,!submodules/tony/tony-mini,!submodules/tony/tony-core,!submodules/tony/tony-proxy,!submodules/tony/tony-portal,!submodules/tony/tony-azkaban,!submodules/tony/tony-cli" @@ -100,12 +99,6 @@ matrix: - yarn run lint --no-fix && yarn run build env: NAME="Build workbench-web" - # Test workbench-server - - language: java - jdk: "openjdk8" - dist: xenial - env: NAME="Test workbench-server" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMARINE},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl submarine-workbench/workbench-server" TEST_PROJECTS="" - # Test workbench-web - language: java jdk: "openjdk8" diff --git a/bin/common.sh b/bin/common.sh index a9cd845d67..70bfed6bb1 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -42,12 +42,12 @@ if [[ -f "${SUBMARINE_CONF_DIR}/submarine-env.sh" ]]; then . "${SUBMARINE_CONF_DIR}/submarine-env.sh" fi -WORKBENCH_CLASSPATH+=":${SUBMARINE_CONF_DIR}" +SUBMARINE_SERVER_CLASSPATH+=":${SUBMARINE_CONF_DIR}" function add_each_jar_in_dir(){ if [[ -d "${1}" ]]; then for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do - WORKBENCH_CLASSPATH="$jar:$WORKBENCH_CLASSPATH" + SUBMARINE_SERVER_CLASSPATH="$jar:$SUBMARINE_SERVER_CLASSPATH" done fi } @@ -55,14 +55,14 @@ function add_each_jar_in_dir(){ function add_each_jar_in_dir_recursive(){ if [[ -d "${1}" ]]; then for jar in $(find -L "${1}" -type f -name '*jar'); do - WORKBENCH_CLASSPATH="$jar:$WORKBENCH_CLASSPATH" + SUBMARINE_SERVER_CLASSPATH="$jar:$SUBMARINE_SERVER_CLASSPATH" done fi } function add_jar_in_dir(){ if [[ -d "${1}" ]]; then - WORKBENCH_CLASSPATH="${1}/*:${WORKBENCH_CLASSPATH}" + SUBMARINE_SERVER_CLASSPATH="${1}/*:${SUBMARINE_SERVER_CLASSPATH}" fi } @@ -74,11 +74,11 @@ function download_mysql_jdbc_jar(){ MYSQL_JAR_URL="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_VERSION}/mysql-connector-java-${MYSQL_VERSION}.jar" fi echo "Downloading mysql jdbc jar from ${MYSQL_JAR_URL}." - wget ${MYSQL_JAR_URL} -P "${BIN}/../workbench/lib" - echo "Mysql jdbc jar is downloaded and put in the path of workbench/lib." + wget ${MYSQL_JAR_URL} -P "${BIN}/../lib" + echo "Mysql jdbc jar is downloaded and put in the path of submarine/lib." } -JAVA_OPTS+=" ${WORKBENCH_JAVA_OPTS} -Dfile.encoding=UTF-8 ${WORKBENCH_MEM}" +JAVA_OPTS+=" ${SUBMARINE_SERVER_JAVA_OPTS} -Dfile.encoding=UTF-8 ${SUBMARINE_SERVER_MEM}" JAVA_OPTS+=" -Dlog4j.configuration=file://${SUBMARINE_CONF_DIR}/log4j.properties" export JAVA_OPTS diff --git a/bin/workbench-daemon.sh b/bin/submarine-daemon.sh similarity index 67% rename from bin/workbench-daemon.sh rename to bin/submarine-daemon.sh index a828e49d72..01aec11ac2 100755 --- a/bin/workbench-daemon.sh +++ b/bin/submarine-daemon.sh @@ -19,7 +19,7 @@ # description: Start and stop daemon script for. # -USAGE="-e Usage: workbench-daemon.sh {start|stop|restart|status}" +USAGE="-e Usage: submarine-daemon.sh {start|stop|restart|status}" if [ -L ${BASH_SOURCE-$0} ]; then BIN=$(dirname $(readlink "${BASH_SOURCE-$0}")) @@ -33,13 +33,12 @@ GET_MYSQL_JAR=false cd ${BIN}/>/dev/null -WORKBENCH_NAME="Submarine Workbench" -WORKBENCH_LOGFILE="${SUBMARINE_LOG_DIR}/workbench.log" -WORKBENCH_MAIN=org.apache.submarine.server.WorkbenchServer -JAVA_OPTS+=" -Dworkbench.log.file=${WORKBENCH_LOGFILE}" +SUBMARINE_SERVER_NAME="Submarine Server" +SUBMARINE_SERVER_LOGFILE="${SUBMARINE_LOG_DIR}/submarine.log" +SUBMARINE_SERVER_MAIN=org.apache.submarine.server.SubmarineServer +JAVA_OPTS+=" -Dsubmarine.log.file=${SUBMARINE_SERVER_LOGFILE}" -add_jar_in_dir "${BIN}/../workbench" -add_jar_in_dir "${BIN}/../workbench/lib" +add_jar_in_dir "${BIN}/../lib" function initialize_default_directories() { if [[ ! -d "${SUBMARINE_LOG_DIR}" ]]; then @@ -48,8 +47,8 @@ function initialize_default_directories() { fi } -function found_workbench_server_pid() { - process='WorkbenchServer'; +function found_submarine_server_pid() { + process='SubmarineServer'; RUNNING_PIDS=$(ps x | grep ${process} | grep -v grep | awk '{print $1}'); if [[ -z "${RUNNING_PIDS}" ]]; then @@ -57,17 +56,17 @@ function found_workbench_server_pid() { fi if ! kill -0 ${RUNNING_PIDS} > /dev/null 2>&1; then - echo "${WORKBENCH_NAME} running but process is dead" + echo "${SUBMARINE_SERVER_NAME} running but process is dead" fi echo "${RUNNING_PIDS}" } -function wait_for_workbench_to_die() { +function wait_for_submarine_server_to_die() { local pid local count - pid=`found_workbench_server_pid` + pid=`found_submarine_server_pid` timeout=10 count=0 timeoutTime=$(date "+%s") @@ -98,9 +97,9 @@ function check_jdbc_jar() { if [[ ${GET_MYSQL_JAR} = true ]]; then download_mysql_jdbc_jar else - echo -e "\\033[31mError: There is no mysql jdbc jar in workbench/lib.\\033[0m" - echo -e "\\033[31mPlease download a mysql jdbc jar and put it under workbench/lib manually.\\033[0m" - echo -e "\\033[31mOr add a parameter getMysqlJar, like this:\n./bin/workbench-daemon.sh start getMysqlJar\\033[0m" + echo -e "\\033[31mError: There is no mysql jdbc jar in lib.\\033[0m" + echo -e "\\033[31mPlease download a mysql jdbc jar and put it under lib manually.\\033[0m" + echo -e "\\033[31mOr add a parameter getMysqlJar, like this:\n./bin/submarine-daemon.sh start getMysqlJar\\033[0m" echo -e "\\033[31mIt would download mysql jdbc jar automatically.\\033[0m" exit 1 fi @@ -111,53 +110,53 @@ function check_jdbc_jar() { function start() { local pid - pid=`found_workbench_server_pid` + pid=`found_submarine_server_pid` if [[ ! -z "$pid" && "$pid" != 0 ]]; then - echo "${WORKBENCH_NAME}:${pid} is already running" + echo "${SUBMARINE_SERVER_NAME}:${pid} is already running" return 0; fi - check_jdbc_jar "${BIN}/../workbench/lib" - + check_jdbc_jar "${BIN}/../lib" + initialize_default_directories - echo "WORKBENCH_CLASSPATH: ${WORKBENCH_CLASSPATH}" >> "${WORKBENCH_LOGFILE}" + echo "SUBMARINE_SERVER_CLASSPATH: ${SUBMARINE_SERVER_CLASSPATH}" >> "${SUBMARINE_SERVER_LOGFILE}" - nohup $JAVA_RUNNER $JAVA_OPTS -cp $WORKBENCH_CLASSPATH $WORKBENCH_MAIN >> "${WORKBENCH_LOGFILE}" 2>&1 < /dev/null & + nohup $JAVA_RUNNER $JAVA_OPTS -cp $SUBMARINE_SERVER_CLASSPATH $SUBMARINE_SERVER_MAIN >> "${SUBMARINE_SERVER_LOGFILE}" 2>&1 < /dev/null & pid=$! if [[ ! -z "${pid}" ]]; then - echo "${WORKBENCH_NAME} start" + echo "${SUBMARINE_SERVER_NAME} start" return 1; fi } function stop() { local pid - pid=`found_workbench_server_pid` + pid=`found_submarine_server_pid` if [[ -z "$pid" ]]; then - echo "${WORKBENCH_NAME} is not running" + echo "${SUBMARINE_SERVER_NAME} is not running" return 0; else # submarine workbench daemon kill - wait_for_workbench_to_die - echo "${WORKBENCH_NAME} stop" + wait_for_submarine_server_to_die + echo "${SUBMARINE_SERVER_NAME} stop" fi } -function find_workbench_process() { +function find_submarine_server_process() { local pid - pid=`found_workbench_server_pid` + pid=`found_submarine_server_pid` if [[ -z "$pid" ]]; then - echo "${WORKBENCH_NAME} is not running" + echo "${SUBMARINE_SERVER_NAME} is not running" return 1 else if ! kill -0 ${pid} > /dev/null 2>&1; then - echo "${WORKBENCH_NAME} running but process is dead" + echo "${SUBMARINE_SERVER_NAME} running but process is dead" return 1 else - echo "${WORKBENCH_NAME} is running" + echo "${SUBMARINE_SERVER_NAME} is running" fi fi } @@ -174,12 +173,12 @@ case "${1}" in stop ;; restart) - echo "${WORKBENCH_NAME} is restarting" >> "${WORKBENCH_LOGFILE}" + echo "${SUBMARINE_SERVER_NAME} is restarting" >> "${SUBMARINE_SERVER_LOGFILE}" stop start ;; status) - find_workbench_process + find_submarine_server_process ;; *) echo ${USAGE} diff --git a/bin/submarine-env.sh b/bin/submarine-env.sh new file mode 100644 index 0000000000..bb78ebbfbd --- /dev/null +++ b/bin/submarine-env.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# export JAVA_HOME=java + +# export SUBMARINE_SERVER_JAVA_OPTS +# export SUBMARINE_SERVER_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m" diff --git a/conf/submarine-env.sh.template b/conf/submarine-env.sh.template index 71aa272ea4..bb78ebbfbd 100644 --- a/conf/submarine-env.sh.template +++ b/conf/submarine-env.sh.template @@ -18,5 +18,5 @@ # export JAVA_HOME=java -# export WORKBENCH_JAVA_OPTS -# export WORKBENCH_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m" +# export SUBMARINE_SERVER_JAVA_OPTS +# export SUBMARINE_SERVER_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m" diff --git a/conf/submarine-site.xml b/conf/submarine-site.xml index a8e7794225..ac2eb710f9 100755 --- a/conf/submarine-site.xml +++ b/conf/submarine-site.xml @@ -20,82 +20,82 @@ - submarine.runtime.class - org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory - RuntimeFactory for Submarine jobs + submarine.cluster.addr + + submarine cluster address list, e.g. ip1:port1;ip2:port2;ip3:port3 - workbench.server.addr + submarine.server.addr 0.0.0.0 Server address - workbench.server.port + submarine.server.port 8080 Server port. - workbench.ssl + submarine.server.ssl false - Should SSL be used by the workbench servers? + Should SSL be used by the submarine server? - workbench.server.ssl.port + submarine.server.ssl.port 8443 Server ssl port. (used when ssl property is set to true) - workbench.ssl.client.auth + submarine.server.ssl.client.auth false Should client authentication be used for SSL connections? - workbench.ssl.keystore.path + submarine.server.ssl.keystore.path keystore Path to keystore relative to submarine configuration directory - workbench.ssl.keystore.type + submarine.server.ssl.keystore.type JKS The format of the given keystore (e.g. JKS or PKCS12) - workbench.ssl.keystore.password + submarine.server.ssl.keystore.password change me Keystore password. Can be obfuscated by the Jetty Password tool - workbench.ssl.truststore.path + submarine.server.ssl.truststore.path truststore Path to truststore relative to submarine configuration directory. Defaults to the keystore path - workbench.ssl.truststore.type + submarine.server.ssl.truststore.type JKS The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type - workbench.ssl.truststore.path + submarine.server.ssl.truststore.path truststore Path to truststore relative to submarine configuration directory. Defaults to the keystore path - workbench.ssl.truststore.type + submarine.server.ssl.truststore.type JKS The format of the given truststore (e.g. JKS or PKCS12). Defaults to the same type as the keystore type + + junit + junit + ${junit.version} + test + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson-annotations.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + + com.fasterxml.jackson.core + jackson-annotations + + + + + + com.google.code.gson + gson + ${gson.version} + + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + + org.mybatis.generator + mybatis-generator-core + ${mybatis-generator.version} + + + commons-io + commons-io + test + + + commons-httpclient + commons-httpclient + ${commons-httpclient.version} + + + commons-logging + commons-logging + + + test + + + + org.eclipse.jgit + org.eclipse.jgit + ${jgit.version} + + + org.slf4j + slf4j-api + + + + + + + submarine-${artifactId}-${project.version} + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies-runtime + package + + copy-dependencies + + + runtime + + + + copy-dependencies-system + package + + copy-dependencies + + + system + true + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + + + + + maven-enforcer-plugin + + + + diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java deleted file mode 100644 index 607d4f9dd5..0000000000 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/JobServer.java +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.submarine.jobserver; - - -import org.eclipse.jetty.http.HttpVersion; -import org.eclipse.jetty.server.HttpConfiguration; -import org.eclipse.jetty.server.HttpConnectionFactory; -import org.eclipse.jetty.server.SecureRequestCustomizer; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.SslConnectionFactory; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; -import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.eclipse.jetty.util.thread.QueuedThreadPool; -import org.eclipse.jetty.util.thread.ThreadPool; -import org.glassfish.jersey.servlet.ServletContainer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.submarine.commons.utils.SubmarineConfiguration; - - -/** - * The ml job server. It will load the classes in rest package when - * bootstrap with related configurable settings. - * */ -public class JobServer { - - private static final Logger LOG = LoggerFactory.getLogger(JobServer.class); - - private SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); - - private Server jobServer; - - public void start() { - ServletContextHandler context = new - ServletContextHandler(ServletContextHandler.SESSIONS); - context.setContextPath("/"); - setupServer(conf);; - jobServer.setHandler(context); - - // Job API servlet - ServletHolder apiServlet = context.addServlet(ServletContainer.class, - conf.getJobServerUrlPrefix()); - apiServlet.setInitOrder(1); - apiServlet.setInitParameter("jersey.config.server.provider.packages", - "org.apache.submarine.jobserver.rest"); - - try { - jobServer.start(); - LOG.info("Submarine job server started"); - jobServer.join(); - } catch (Exception e) { - LOG.error("Submarine job server failed to start"); - e.printStackTrace(); - } finally { - jobServer.destroy(); - LOG.info("Submarine job server stopped"); - } - } - - public static void main(String[] args) throws Exception { - new JobServer().start(); - } - - private Server setupServer(SubmarineConfiguration conf) { - ThreadPool threadPool = - new QueuedThreadPool( - conf.getInt(SubmarineConfiguration - .ConfVars.SERVER_JETTY_THREAD_POOL_MAX), - conf.getInt(SubmarineConfiguration - .ConfVars.SERVER_JETTY_THREAD_POOL_MIN), - conf.getInt(SubmarineConfiguration - .ConfVars.SERVER_JETTY_THREAD_POOL_TIMEOUT)); - jobServer = new Server(threadPool); - ServerConnector connector; - - if (conf.isJobServerSslEnabled()) { - LOG.debug("Enabling SSL for submarine job server on port " - + conf.getServerSslPort()); - HttpConfiguration httpConfig = new HttpConfiguration(); - httpConfig.setSecureScheme("https"); - httpConfig.setSecurePort(conf.getServerSslPort()); - httpConfig.setOutputBufferSize(32768); - httpConfig.setResponseHeaderSize(8192); - httpConfig.setSendServerVersion(true); - - HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig); - SecureRequestCustomizer src = new SecureRequestCustomizer(); - httpsConfig.addCustomizer(src); - - connector = new ServerConnector( - jobServer, - new SslConnectionFactory(getSslContextFactory(conf), - HttpVersion.HTTP_1_1.asString()), - new HttpConnectionFactory(httpsConfig)); - } else { - connector = new ServerConnector(jobServer); - } - - configureRequestHeaderSize(conf, connector); - // Set some timeout options to make debugging easier. - int timeout = 1000 * 30; - connector.setIdleTimeout(timeout); - connector.setSoLingerTime(-1); - connector.setHost(conf.getJobServerAddress()); - if (conf.useSsl()) { - connector.setPort(conf.getJobServerSslPort()); - } else { - connector.setPort(conf.getJobServerPort()); - } - - jobServer.addConnector(connector); - return jobServer; - } - - private static SslContextFactory getSslContextFactory( - SubmarineConfiguration conf) { - SslContextFactory sslContextFactory = new SslContextFactory(); - - // Set keystore - sslContextFactory.setKeyStorePath(conf.getKeyStorePath()); - sslContextFactory.setKeyStoreType(conf.getKeyStoreType()); - sslContextFactory.setKeyStorePassword(conf.getKeyStorePassword()); - sslContextFactory.setKeyManagerPassword(conf.getKeyManagerPassword()); - - if (conf.useClientAuth()) { - sslContextFactory.setNeedClientAuth(conf.useClientAuth()); - - // Set truststore - sslContextFactory.setTrustStorePath(conf.getTrustStorePath()); - sslContextFactory.setTrustStoreType(conf.getTrustStoreType()); - sslContextFactory.setTrustStorePassword(conf.getTrustStorePassword()); - } - - return sslContextFactory; - } - - private static void configureRequestHeaderSize( - SubmarineConfiguration conf, ServerConnector connector) { - HttpConnectionFactory cf = - (HttpConnectionFactory) connector - .getConnectionFactory(HttpVersion.HTTP_1_1.toString()); - int requestHeaderSize = conf.getJettyRequestHeaderSize(); - cf.getHttpConfiguration().setRequestHeaderSize(requestHeaderSize); - } - -} diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java similarity index 90% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java index ad1dc749bc..a5bb2085e1 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/WorkbenchServer.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java @@ -19,7 +19,7 @@ package org.apache.submarine.server; import org.apache.log4j.PropertyConfigurator; -import org.apache.submarine.websocket.NotebookServer; +import org.apache.submarine.server.workbench.websocket.NotebookServer; import org.apache.submarine.commons.cluster.ClusterServer; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.HttpConfiguration; @@ -53,8 +53,8 @@ import javax.inject.Singleton; import java.io.File; -public class WorkbenchServer extends ResourceConfig { - private static final Logger LOG = LoggerFactory.getLogger(WorkbenchServer.class); +public class SubmarineServer extends ResourceConfig { + private static final Logger LOG = LoggerFactory.getLogger(SubmarineServer.class); public static Server jettyWebServer; public static ServiceLocator sharedServiceLocator; @@ -65,11 +65,11 @@ public static void main(String[] args) throws InterruptedException { PropertyConfigurator.configure(ClassLoader.getSystemResource("log4j.properties")); final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); - LOG.info("Workbench server Host: " + conf.getServerAddress()); + LOG.info("Submarine server Host: " + conf.getServerAddress()); if (conf.useSsl() == false) { - LOG.info("Workbench server Port: " + conf.getServerPort()); + LOG.info("Submarine server Port: " + conf.getServerPort()); } else { - LOG.info("Workbench server SSL Port: " + conf.getServerSslPort()); + LOG.info("Submarine server SSL Port: " + conf.getServerSslPort()); } jettyWebServer = setupJettyServer(conf); @@ -95,6 +95,7 @@ protected void configure() { }); setupRestApiContextHandler(webApp, conf); + // Notebook server setupNotebookServer(webApp, conf, sharedServiceLocator); @@ -105,8 +106,9 @@ protected void configure() { } @Inject - public WorkbenchServer() { - packages("org.apache.submarine.rest"); + public SubmarineServer() { + packages("org.apache.submarine.server.workbench.rest", + "org.apache.submarine.server.jobserver.rest.api"); } private static void startServer() throws InterruptedException { @@ -123,7 +125,7 @@ private static void startServer() throws InterruptedException { .addShutdownHook( new Thread( () -> { - LOG.info("Shutting down Submarine Workbench Server ... "); + LOG.info("Shutting down Submarine Server ... "); try { jettyWebServer.stop(); Thread.sleep(3000); @@ -140,7 +142,7 @@ private static void setupRestApiContextHandler(WebAppContext webapp, SubmarineCo final ServletHolder servletHolder = new ServletHolder(new org.glassfish.jersey.servlet.ServletContainer()); - servletHolder.setInitParameter("javax.ws.rs.Application", WorkbenchServer.class.getName()); + servletHolder.setInitParameter("javax.ws.rs.Application", SubmarineServer.class.getName()); servletHolder.setName("rest"); servletHolder.setForcedPath("rest"); webapp.setSessionHandler(new SessionHandler()); @@ -174,9 +176,9 @@ private static WebAppContext setupWebAppContext(ContextHandlerCollection context private static Server setupJettyServer(SubmarineConfiguration conf) { ThreadPool threadPool = - new QueuedThreadPool(conf.getInt(ConfVars.SERVER_JETTY_THREAD_POOL_MAX), - conf.getInt(ConfVars.SERVER_JETTY_THREAD_POOL_MIN), - conf.getInt(ConfVars.SERVER_JETTY_THREAD_POOL_TIMEOUT)); + new QueuedThreadPool(conf.getInt(ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX), + conf.getInt(ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN), + conf.getInt(ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT)); final Server server = new Server(threadPool); ServerConnector connector; @@ -229,7 +231,7 @@ private static void setupNotebookServer(WebAppContext webapp, } private static void setupClusterServer() { - if (conf.workbenchIsClusterMode()) { + if (conf.isClusterMode()) { ClusterServer clusterServer = ClusterServer.getInstance(); clusterServer.start(); } diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/api/JobApi.java similarity index 91% rename from submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/api/JobApi.java index 8b842f49d1..039d196be2 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/api/JobApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/api/JobApi.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.submarine.jobserver.rest.api; +package org.apache.submarine.server.jobserver.rest.api; -import org.apache.submarine.jobserver.rest.dao.JsonResponse; -import org.apache.submarine.jobserver.rest.dao.MLJobSpec; -import org.apache.submarine.jobserver.rest.dao.RestConstants; +import org.apache.submarine.server.jobserver.rest.dao.JsonResponse; +import org.apache.submarine.server.jobserver.rest.dao.MLJobSpec; +import org.apache.submarine.server.jobserver.rest.dao.RestConstants; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/Component.java similarity index 96% rename from submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/Component.java index a2c1458d52..e5b39efc0f 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/Component.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/Component.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.submarine.jobserver.rest.dao; +package org.apache.submarine.server.jobserver.rest.dao; /** diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/EnvVaraible.java similarity index 95% rename from submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/EnvVaraible.java index 24a4ae2aca..b8388ff616 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/EnvVaraible.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/EnvVaraible.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.submarine.jobserver.rest.dao; +package org.apache.submarine.server.jobserver.rest.dao; // A process level environment variable. public class EnvVaraible { diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/JsonExclusionStrategy.java similarity index 95% rename from submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/JsonExclusionStrategy.java index c24e413429..9218ac402e 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonExclusionStrategy.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/JsonExclusionStrategy.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.submarine.jobserver.rest.dao; +package org.apache.submarine.server.jobserver.rest.dao; import com.google.gson.ExclusionStrategy; import com.google.gson.FieldAttributes; diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/JsonResponse.java similarity index 93% rename from submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/JsonResponse.java index 46726cf24b..e6dcccc497 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/JsonResponse.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/JsonResponse.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.submarine.jobserver.rest.dao; +package org.apache.submarine.server.jobserver.rest.dao; import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; @@ -160,19 +160,7 @@ public String toString() { .serializeNulls().create(); } -// boolean haveDictAnnotation = false; -// try { -// if (null != getResult()) { -// haveDictAnnotation = DictAnnotation.parseDictAnnotation(getResult()); -// } -// } catch (Exception e) { -// LOG.error(e.getMessage(), e); -// } - String json = safeGson.toJson(this); -// if (haveDictAnnotation) { -// json = json.replaceAll(CGLIB_PROPERTY_PREFIX, ""); -// } return json; } diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/MLJobSpec.java similarity index 98% rename from submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/MLJobSpec.java index 26f7fa823a..0e64b710a6 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/MLJobSpec.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/MLJobSpec.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.submarine.jobserver.rest.dao; +package org.apache.submarine.server.jobserver.rest.dao; /** * The machine learning job spec the submarine job server can accept. diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/RestConstants.java similarity index 70% rename from submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/RestConstants.java index a33a9314e1..360d86cf99 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/dao/RestConstants.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/dao/RestConstants.java @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.submarine.jobserver.rest.dao; +package org.apache.submarine.server.jobserver.rest.dao; public class RestConstants { - public final static String V1 = "v1"; - public final static String JOBS = "jobs"; - public final static String JOB_ID = "id"; - public final static String PING = "ping"; - public final static String MEDIA_TYPE_YAML = "application/yaml"; - public final static String CHARSET_UTF8 = "charset=utf-8"; + public static final String V1 = "v1"; + public static final String JOBS = "jobs"; + public static final String JOB_ID = "id"; + public static final String PING = "ping"; + public static final String MEDIA_TYPE_YAML = "application/yaml"; + public static final String CHARSET_UTF8 = "charset=utf-8"; } diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/provider/YamlEntityProvider.java similarity index 97% rename from submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/provider/YamlEntityProvider.java index 3343cc89e2..8250ce0e84 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/jobserver/rest/provider/YamlEntityProvider.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/jobserver/rest/provider/YamlEntityProvider.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.submarine.jobserver.rest.provider; +package org.apache.submarine.server.jobserver.rest.provider; import org.yaml.snakeyaml.Yaml; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/annotation/Dict.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/annotation/Dict.java similarity index 94% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/annotation/Dict.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/annotation/Dict.java index b354c80590..29f7aa7e68 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/annotation/Dict.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/annotation/Dict.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.annotation; +package org.apache.submarine.server.workbench.annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/annotation/SubmarineApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/annotation/SubmarineApi.java similarity index 95% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/annotation/SubmarineApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/annotation/SubmarineApi.java index b73e8e314b..0a23396b6c 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/annotation/SubmarineApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/annotation/SubmarineApi.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.annotation; +package org.apache.submarine.server.workbench.annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/MyBatisUtil.java similarity index 98% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/MyBatisUtil.java index 94cb76a503..dd9a935f2b 100755 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/MyBatisUtil.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/MyBatisUtil.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database; +package org.apache.submarine.server.workbench.database; import org.apache.ibatis.io.Resources; import org.apache.ibatis.session.SqlSession; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/BaseEntity.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/BaseEntity.java similarity index 95% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/BaseEntity.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/BaseEntity.java index ba301fad88..1ed11a08b9 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/BaseEntity.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/BaseEntity.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.annotations.VisibleForTesting; -import org.apache.submarine.database.utils.CustomJsonDateDeserializer; +import org.apache.submarine.server.workbench.database.utils.CustomJsonDateDeserializer; import java.lang.reflect.Field; import java.util.Date; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/Project.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Project.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/Project.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Project.java index 848ab22879..966db2120a 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/Project.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Project.java @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; -import org.apache.submarine.annotation.Dict; +import org.apache.submarine.server.workbench.annotation.Dict; import java.util.ArrayList; import java.util.List; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/ProjectFiles.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/ProjectFiles.java similarity index 95% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/ProjectFiles.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/ProjectFiles.java index 08c367c917..6e64c312aa 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/ProjectFiles.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/ProjectFiles.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; public class ProjectFiles extends BaseEntity { private String projectId; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDept.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDept.java similarity index 97% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDept.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDept.java index 2896690ab8..e0a159c545 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDept.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDept.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; public class SysDept extends BaseEntity { private String deptCode; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDeptSelect.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDeptSelect.java similarity index 97% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDeptSelect.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDeptSelect.java index fe41842b8a..3f2c57e2da 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDeptSelect.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDeptSelect.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; import java.util.ArrayList; import java.util.List; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDeptTree.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDeptTree.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDeptTree.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDeptTree.java index 66c832e490..c8d6231928 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDeptTree.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDeptTree.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; import java.util.ArrayList; import java.util.List; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDict.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDict.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDict.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDict.java index 403fd22c72..572100e4e9 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDict.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDict.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; public class SysDict extends BaseEntity { diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDictItem.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDictItem.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDictItem.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDictItem.java index 1a578c3eec..3067c54874 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysDictItem.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysDictItem.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; public class SysDictItem extends BaseEntity { diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysMessage.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysMessage.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysMessage.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysMessage.java index 8b8f857690..d65a548642 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysMessage.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysMessage.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; public class SysMessage extends BaseEntity { diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysUser.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysUser.java similarity index 93% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysUser.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysUser.java index feb8a9f293..32019c5946 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/SysUser.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/SysUser.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import org.apache.submarine.annotation.Dict; -import org.apache.submarine.database.utils.CustomJsonDateDeserializer; +import org.apache.submarine.server.workbench.annotation.Dict; +import org.apache.submarine.server.workbench.database.utils.CustomJsonDateDeserializer; import java.util.Date; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/Team.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Team.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/Team.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Team.java index c878a0e093..5c9010c47d 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/Team.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/Team.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; import java.util.ArrayList; import java.util.List; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/TeamMember.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/TeamMember.java similarity index 95% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/TeamMember.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/TeamMember.java index 5b13cc6246..41c326b0b1 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/entity/TeamMember.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/entity/TeamMember.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.entity; +package org.apache.submarine.server.workbench.database.entity; public class TeamMember extends BaseEntity { diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/ProjectFilesMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/ProjectFilesMapper.java similarity index 90% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/ProjectFilesMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/ProjectFilesMapper.java index b15b779cad..06b2067cdc 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/ProjectFilesMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/ProjectFilesMapper.java @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; -import org.apache.submarine.database.entity.ProjectFiles; +import org.apache.submarine.server.workbench.database.entity.ProjectFiles; import java.util.List; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/ProjectMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/ProjectMapper.java similarity index 90% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/ProjectMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/ProjectMapper.java index e5ca4dc60e..4604a28b70 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/ProjectMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/ProjectMapper.java @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; import org.apache.ibatis.session.RowBounds; -import org.apache.submarine.database.entity.Project; +import org.apache.submarine.server.workbench.database.entity.Project; import java.util.List; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDeptMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDeptMapper.java similarity index 89% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDeptMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDeptMapper.java index a39634cc01..cfd59e1a15 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDeptMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDeptMapper.java @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; -import org.apache.submarine.database.entity.SysDept; +import org.apache.submarine.server.workbench.database.entity.SysDept; import java.util.List; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDictItemMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDictItemMapper.java similarity index 89% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDictItemMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDictItemMapper.java index c1bbc02f7d..855fddbeaa 100755 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDictItemMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDictItemMapper.java @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; import org.apache.ibatis.session.RowBounds; -import org.apache.submarine.database.entity.SysDictItem; +import org.apache.submarine.server.workbench.database.entity.SysDictItem; import java.util.List; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDictMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDictMapper.java similarity index 89% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDictMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDictMapper.java index 847d471b51..8f3fd59f4d 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysDictMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysDictMapper.java @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; import org.apache.ibatis.session.RowBounds; -import org.apache.submarine.database.entity.SysDict; +import org.apache.submarine.server.workbench.database.entity.SysDict; import java.util.List; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysMessageMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysMessageMapper.java similarity index 89% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysMessageMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysMessageMapper.java index f3cd28f583..6ff4c05b06 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysMessageMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysMessageMapper.java @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; -import org.apache.submarine.database.entity.SysMessage; +import org.apache.submarine.server.workbench.database.entity.SysMessage; public interface SysMessageMapper { int deleteByPrimaryKey(String id); diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysUserMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysUserMapper.java similarity index 90% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysUserMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysUserMapper.java index 10b2c495e1..1d9697ad2c 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SysUserMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SysUserMapper.java @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; import org.apache.ibatis.session.RowBounds; -import org.apache.submarine.database.entity.SysUser; +import org.apache.submarine.server.workbench.database.entity.SysUser; import java.util.List; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SystemMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SystemMapper.java similarity index 93% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SystemMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SystemMapper.java index b624220afd..0b5ffe6041 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/SystemMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/SystemMapper.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/TeamMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/TeamMapper.java similarity index 90% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/TeamMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/TeamMapper.java index 5d877a4e7d..e904913924 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/TeamMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/TeamMapper.java @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; import org.apache.ibatis.session.RowBounds; -import org.apache.submarine.database.entity.Team; +import org.apache.submarine.server.workbench.database.entity.Team; import java.util.List; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/TeamMemberMapper.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/TeamMemberMapper.java similarity index 90% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/TeamMemberMapper.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/TeamMemberMapper.java index 9bcea17091..9f905b6787 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/mappers/TeamMemberMapper.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/mappers/TeamMemberMapper.java @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.mappers; +package org.apache.submarine.server.workbench.database.mappers; -import org.apache.submarine.database.entity.TeamMember; +import org.apache.submarine.server.workbench.database.entity.TeamMember; import java.util.List; import java.util.Map; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/ProjectFilesService.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/ProjectFilesService.java similarity index 85% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/ProjectFilesService.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/ProjectFilesService.java index 9ba8776a37..3eb9e36c0d 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/ProjectFilesService.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/ProjectFilesService.java @@ -16,12 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.service; +package org.apache.submarine.server.workbench.database.service; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.ProjectFiles; -import org.apache.submarine.database.mappers.ProjectFilesMapper; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.ProjectFiles; +import org.apache.submarine.server.workbench.database.mappers.ProjectFilesMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/ProjectService.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/ProjectService.java similarity index 93% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/ProjectService.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/ProjectService.java index 6c434db6f8..6396370226 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/ProjectService.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/ProjectService.java @@ -16,15 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.service; +package org.apache.submarine.server.workbench.database.service; import org.apache.ibatis.session.RowBounds; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.Project; -import org.apache.submarine.database.entity.ProjectFiles; -import org.apache.submarine.database.mappers.ProjectFilesMapper; -import org.apache.submarine.database.mappers.ProjectMapper; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.Project; +import org.apache.submarine.server.workbench.database.entity.ProjectFiles; +import org.apache.submarine.server.workbench.database.mappers.ProjectFilesMapper; +import org.apache.submarine.server.workbench.database.mappers.ProjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysDictItemService.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysDictItemService.java similarity index 81% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysDictItemService.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysDictItemService.java index 3e8b6fcd67..dea66c22f0 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysDictItemService.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysDictItemService.java @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.service; +package org.apache.submarine.server.workbench.database.service; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.SysDictItem; -import org.apache.submarine.database.mappers.SysDictItemMapper; -import org.apache.submarine.rest.SysDictRestApi; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.SysDictItem; +import org.apache.submarine.server.workbench.database.mappers.SysDictItemMapper; +import org.apache.submarine.server.workbench.rest.SysDictRestApi; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysMessageService.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysMessageService.java similarity index 83% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysMessageService.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysMessageService.java index 54cbb9d345..de6dd728d5 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysMessageService.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysMessageService.java @@ -16,12 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.service; +package org.apache.submarine.server.workbench.database.service; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.SysMessage; -import org.apache.submarine.database.mappers.SysMessageMapper; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.SysMessage; +import org.apache.submarine.server.workbench.database.mappers.SysMessageMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysUserService.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysUserService.java similarity index 94% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysUserService.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysUserService.java index 26f0fd912c..0c714ae84d 100755 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/SysUserService.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/SysUserService.java @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.service; +package org.apache.submarine.server.workbench.database.service; import org.apache.ibatis.session.RowBounds; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.SysUser; -import org.apache.submarine.database.mappers.SysUserMapper; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.SysUser; +import org.apache.submarine.server.workbench.database.mappers.SysUserMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/TeamMemberService.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/TeamMemberService.java similarity index 90% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/TeamMemberService.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/TeamMemberService.java index e70f81480f..aea9fb2e48 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/TeamMemberService.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/TeamMemberService.java @@ -16,12 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.service; +package org.apache.submarine.server.workbench.database.service; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.TeamMember; -import org.apache.submarine.database.mappers.TeamMemberMapper; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.TeamMember; +import org.apache.submarine.server.workbench.database.mappers.TeamMemberMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/TeamService.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/TeamService.java similarity index 93% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/TeamService.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/TeamService.java index 29fa3be535..a365eef6b1 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/service/TeamService.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/service/TeamService.java @@ -16,15 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.service; +package org.apache.submarine.server.workbench.database.service; import org.apache.ibatis.session.RowBounds; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.Team; -import org.apache.submarine.database.entity.TeamMember; -import org.apache.submarine.database.mappers.TeamMapper; -import org.apache.submarine.database.mappers.TeamMemberMapper; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.Team; +import org.apache.submarine.server.workbench.database.entity.TeamMember; +import org.apache.submarine.server.workbench.database.mappers.TeamMapper; +import org.apache.submarine.server.workbench.database.mappers.TeamMemberMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/CustomJsonDateDeserializer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/CustomJsonDateDeserializer.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/CustomJsonDateDeserializer.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/CustomJsonDateDeserializer.java index 3ed7e1aedc..30cb6f0f68 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/CustomJsonDateDeserializer.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/CustomJsonDateDeserializer.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.utils; +package org.apache.submarine.server.workbench.database.utils; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/DepartmentUtil.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DepartmentUtil.java similarity index 93% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/DepartmentUtil.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DepartmentUtil.java index fd6f65bdc4..479ac00386 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/DepartmentUtil.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DepartmentUtil.java @@ -16,11 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.utils; +package org.apache.submarine.server.workbench.database.utils; -import org.apache.submarine.database.entity.SysDeptSelect; -import org.apache.submarine.database.entity.SysDeptTree; -import org.apache.submarine.database.entity.SysDept; +import org.apache.submarine.server.workbench.database.entity.SysDeptSelect; +import org.apache.submarine.server.workbench.database.entity.SysDeptTree; +import org.apache.submarine.server.workbench.database.entity.SysDept; import java.util.ArrayList; import java.util.List; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/DictAnnotation.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DictAnnotation.java similarity index 94% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/DictAnnotation.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DictAnnotation.java index 503bf1b1b6..025185cb52 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/DictAnnotation.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/DictAnnotation.java @@ -16,15 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.utils; +package org.apache.submarine.server.workbench.database.utils; import net.sf.cglib.beans.BeanGenerator; import net.sf.cglib.beans.BeanMap; import org.apache.commons.lang.StringUtils; -import org.apache.submarine.annotation.Dict; -import org.apache.submarine.database.entity.SysDictItem; -import org.apache.submarine.database.service.SysDictItemService; -import org.apache.submarine.server.JsonResponse.ListResult; +import org.apache.submarine.server.workbench.annotation.Dict; +import org.apache.submarine.server.workbench.database.entity.SysDictItem; +import org.apache.submarine.server.workbench.database.service.SysDictItemService; +import org.apache.submarine.server.workbench.server.JsonResponse.ListResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/GitUtils.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/GitUtils.java similarity index 99% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/GitUtils.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/GitUtils.java index 0c0c58a592..666648b5f3 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/GitUtils.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/GitUtils.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.utils; +package org.apache.submarine.server.workbench.database.utils; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ListBranchCommand; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/HttpRequestUtil.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/HttpRequestUtil.java similarity index 98% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/HttpRequestUtil.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/HttpRequestUtil.java index fb392c49b4..f374146594 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/HttpRequestUtil.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/HttpRequestUtil.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.utils; +package org.apache.submarine.server.workbench.database.utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/MybatisGeneratorMain.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/MybatisGeneratorMain.java similarity index 95% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/MybatisGeneratorMain.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/MybatisGeneratorMain.java index 2d86707df6..68eb63add6 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/database/utils/MybatisGeneratorMain.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/database/utils/MybatisGeneratorMain.java @@ -16,9 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.database.utils; +package org.apache.submarine.server.workbench.database.utils; -import org.apache.submarine.database.service.TeamService; +import org.apache.submarine.server.workbench.database.service.TeamService; import org.mybatis.generator.api.MyBatisGenerator; import org.mybatis.generator.config.Configuration; import org.mybatis.generator.config.xml.ConfigurationParser; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Action.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Action.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Action.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Action.java index bd21959f7f..0ff63702ca 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Action.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Action.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.entity; +package org.apache.submarine.server.workbench.entity; public class Action { private String action; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Permission.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Permission.java similarity index 98% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Permission.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Permission.java index 6dd5471dff..ba62cd327b 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Permission.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Permission.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.entity; +package org.apache.submarine.server.workbench.entity; import java.util.List; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Role.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Role.java similarity index 98% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Role.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Role.java index 19b7966030..1944f7378f 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/Role.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/Role.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.entity; +package org.apache.submarine.server.workbench.entity; import java.util.List; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/User.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/User.java similarity index 98% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/User.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/User.java index 6c5eadc527..8e89188838 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/User.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/User.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.entity; +package org.apache.submarine.server.workbench.entity; public class User { private final String id; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/UserInfo.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/UserInfo.java similarity index 98% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/UserInfo.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/UserInfo.java index 6ece630e72..c2d18dc8e2 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/entity/UserInfo.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/entity/UserInfo.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.entity; +package org.apache.submarine.server.workbench.entity; public class UserInfo { private final String id; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/LoginRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/LoginRestApi.java similarity index 86% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/LoginRestApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/LoginRestApi.java index f0b89c7211..3149735d09 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/LoginRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/LoginRestApi.java @@ -16,16 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.rest; +package org.apache.submarine.server.workbench.rest; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.annotation.SubmarineApi; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.SysUser; -import org.apache.submarine.database.mappers.SysUserMapper; -import org.apache.submarine.server.JsonResponse; +import org.apache.submarine.server.workbench.annotation.SubmarineApi; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.SysUser; +import org.apache.submarine.server.workbench.database.mappers.SysUserMapper; +import org.apache.submarine.server.workbench.server.JsonResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/ProjectRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/ProjectRestApi.java similarity index 91% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/ProjectRestApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/ProjectRestApi.java index 8d98b08c5a..28ea7f0d4f 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/ProjectRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/ProjectRestApi.java @@ -16,14 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.rest; +package org.apache.submarine.server.workbench.rest; import com.github.pagehelper.PageInfo; -import org.apache.submarine.annotation.SubmarineApi; -import org.apache.submarine.database.entity.Project; -import org.apache.submarine.database.service.ProjectService; -import org.apache.submarine.server.JsonResponse; -import org.apache.submarine.server.JsonResponse.ListResult; +import org.apache.submarine.server.workbench.annotation.SubmarineApi; +import org.apache.submarine.server.workbench.database.entity.Project; +import org.apache.submarine.server.workbench.database.service.ProjectService; +import org.apache.submarine.server.workbench.server.JsonResponse; +import org.apache.submarine.server.workbench.server.JsonResponse.ListResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDeptRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDeptRestApi.java similarity index 92% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDeptRestApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDeptRestApi.java index f24954b187..891d9f2851 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDeptRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDeptRestApi.java @@ -16,20 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.rest; +package org.apache.submarine.server.workbench.rest; import com.github.pagehelper.PageInfo; import org.apache.commons.lang.StringUtils; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.annotation.SubmarineApi; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.SysDeptSelect; -import org.apache.submarine.database.entity.SysDeptTree; -import org.apache.submarine.database.entity.SysDept; -import org.apache.submarine.database.mappers.SysDeptMapper; -import org.apache.submarine.database.utils.DepartmentUtil; -import org.apache.submarine.server.JsonResponse; -import org.apache.submarine.server.JsonResponse.ListResult; +import org.apache.submarine.server.workbench.annotation.SubmarineApi; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.SysDeptSelect; +import org.apache.submarine.server.workbench.database.entity.SysDeptTree; +import org.apache.submarine.server.workbench.database.entity.SysDept; +import org.apache.submarine.server.workbench.database.mappers.SysDeptMapper; +import org.apache.submarine.server.workbench.database.utils.DepartmentUtil; +import org.apache.submarine.server.workbench.server.JsonResponse; +import org.apache.submarine.server.workbench.server.JsonResponse.ListResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDictItemRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDictItemRestApi.java similarity index 93% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDictItemRestApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDictItemRestApi.java index a79bc2669b..79d0f9f3cb 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDictItemRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDictItemRestApi.java @@ -16,19 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.rest; +package org.apache.submarine.server.workbench.rest; import com.github.pagehelper.PageInfo; import com.google.gson.Gson; import org.apache.ibatis.session.RowBounds; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.annotation.SubmarineApi; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.SysDictItem; -import org.apache.submarine.database.mappers.SysDictItemMapper; -import org.apache.submarine.database.service.SysDictItemService; -import org.apache.submarine.server.JsonResponse; -import org.apache.submarine.server.JsonResponse.ListResult; +import org.apache.submarine.server.workbench.annotation.SubmarineApi; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.SysDictItem; +import org.apache.submarine.server.workbench.database.mappers.SysDictItemMapper; +import org.apache.submarine.server.workbench.database.service.SysDictItemService; +import org.apache.submarine.server.workbench.server.JsonResponse; +import org.apache.submarine.server.workbench.server.JsonResponse.ListResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDictRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDictRestApi.java similarity index 93% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDictRestApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDictRestApi.java index 1217ae73b3..0dfcbf779e 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysDictRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysDictRestApi.java @@ -16,18 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.rest; +package org.apache.submarine.server.workbench.rest; import com.github.pagehelper.PageInfo; import com.google.gson.Gson; import org.apache.ibatis.session.RowBounds; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.annotation.SubmarineApi; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.SysDict; -import org.apache.submarine.database.mappers.SysDictMapper; -import org.apache.submarine.server.JsonResponse; -import org.apache.submarine.server.JsonResponse.ListResult; +import org.apache.submarine.server.workbench.annotation.SubmarineApi; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.SysDict; +import org.apache.submarine.server.workbench.database.mappers.SysDictMapper; +import org.apache.submarine.server.workbench.server.JsonResponse; +import org.apache.submarine.server.workbench.server.JsonResponse.ListResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysUserRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysUserRestApi.java similarity index 92% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysUserRestApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysUserRestApi.java index c1d6668add..6107831976 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SysUserRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SysUserRestApi.java @@ -16,20 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.rest; +package org.apache.submarine.server.workbench.rest; import com.github.pagehelper.PageInfo; import com.google.gson.Gson; -import org.apache.submarine.annotation.SubmarineApi; -import org.apache.submarine.database.entity.SysDept; -import org.apache.submarine.database.entity.SysUser; -import org.apache.submarine.database.service.SysUserService; -import org.apache.submarine.entity.Action; -import org.apache.submarine.entity.Permission; -import org.apache.submarine.entity.Role; -import org.apache.submarine.entity.UserInfo; -import org.apache.submarine.server.JsonResponse; -import org.apache.submarine.server.JsonResponse.ListResult; +import org.apache.submarine.server.workbench.annotation.SubmarineApi; +import org.apache.submarine.server.workbench.database.entity.SysDept; +import org.apache.submarine.server.workbench.database.entity.SysUser; +import org.apache.submarine.server.workbench.database.service.SysUserService; +import org.apache.submarine.server.workbench.entity.Action; +import org.apache.submarine.server.workbench.entity.Permission; +import org.apache.submarine.server.workbench.entity.Role; +import org.apache.submarine.server.workbench.entity.UserInfo; +import org.apache.submarine.server.workbench.server.JsonResponse; +import org.apache.submarine.server.workbench.server.JsonResponse.ListResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SystemRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SystemRestApi.java similarity index 88% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SystemRestApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SystemRestApi.java index 3f07ae4a7c..3e26689443 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/SystemRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/SystemRestApi.java @@ -16,18 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.rest; +package org.apache.submarine.server.workbench.rest; import com.github.pagehelper.PageInfo; import org.apache.commons.lang.StringUtils; import org.apache.ibatis.session.SqlSession; -import org.apache.submarine.annotation.SubmarineApi; -import org.apache.submarine.database.MyBatisUtil; -import org.apache.submarine.database.entity.SysUser; -import org.apache.submarine.database.mappers.SystemMapper; -import org.apache.submarine.database.service.SysUserService; -import org.apache.submarine.server.JsonResponse; -import org.apache.submarine.server.JsonResponse.ListResult; +import org.apache.submarine.server.workbench.annotation.SubmarineApi; +import org.apache.submarine.server.workbench.database.MyBatisUtil; +import org.apache.submarine.server.workbench.database.entity.SysUser; +import org.apache.submarine.server.workbench.database.mappers.SystemMapper; +import org.apache.submarine.server.workbench.database.service.SysUserService; +import org.apache.submarine.server.workbench.server.JsonResponse; +import org.apache.submarine.server.workbench.server.JsonResponse.ListResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/TeamRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/TeamRestApi.java similarity index 92% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/TeamRestApi.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/TeamRestApi.java index 028be6b9e0..6eae8aff17 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/rest/TeamRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/rest/TeamRestApi.java @@ -16,14 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.rest; +package org.apache.submarine.server.workbench.rest; import com.github.pagehelper.PageInfo; -import org.apache.submarine.annotation.SubmarineApi; -import org.apache.submarine.database.entity.Team; -import org.apache.submarine.database.service.TeamService; -import org.apache.submarine.server.JsonResponse; -import org.apache.submarine.server.JsonResponse.ListResult; +import org.apache.submarine.server.workbench.annotation.SubmarineApi; +import org.apache.submarine.server.workbench.database.entity.Team; +import org.apache.submarine.server.workbench.database.service.TeamService; +import org.apache.submarine.server.workbench.server.JsonResponse; +import org.apache.submarine.server.workbench.server.JsonResponse.ListResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/JsonExclusionStrategy.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/server/JsonExclusionStrategy.java similarity index 95% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/JsonExclusionStrategy.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/server/JsonExclusionStrategy.java index 9d92ca5f5c..c01966a93e 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/JsonExclusionStrategy.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/server/JsonExclusionStrategy.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.server; +package org.apache.submarine.server.workbench.server; import com.google.gson.ExclusionStrategy; import com.google.gson.FieldAttributes; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/JsonResponse.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/server/JsonResponse.java similarity index 97% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/JsonResponse.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/server/JsonResponse.java index a45da02c1c..4f589604d4 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/server/JsonResponse.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/server/JsonResponse.java @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.submarine.server; +package org.apache.submarine.server.workbench.server; import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.TypeAdapter; -import org.apache.submarine.database.utils.DictAnnotation; +import org.apache.submarine.server.workbench.database.utils.DictAnnotation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/utils/TestUtils.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/utils/TestUtils.java similarity index 91% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/utils/TestUtils.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/utils/TestUtils.java index 8142f15e94..b821908810 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/utils/TestUtils.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/utils/TestUtils.java @@ -14,9 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.submarine.utils; +package org.apache.submarine.server.workbench.utils; -import org.apache.submarine.server.WorkbenchServer; +import org.apache.submarine.server.SubmarineServer; import org.glassfish.hk2.api.ServiceLocator; import org.glassfish.hk2.api.ServiceLocatorFactory; @@ -25,7 +25,7 @@ public class TestUtils { public static T getInstance(Class clazz) { checkCalledByTestMethod(); - return getInstance(WorkbenchServer.sharedServiceLocator, clazz); + return getInstance(SubmarineServer.sharedServiceLocator, clazz); } public static void clearInstances() { diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/ConnectionManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/ConnectionManager.java similarity index 98% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/ConnectionManager.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/ConnectionManager.java index 3ed92e7cd4..fbb173d455 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/ConnectionManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/ConnectionManager.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.submarine.websocket; +package org.apache.submarine.server.workbench.websocket; import com.google.common.collect.Sets; import com.google.gson.Gson; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/DateJsonDeserializer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/DateJsonDeserializer.java similarity index 97% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/DateJsonDeserializer.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/DateJsonDeserializer.java index 07885b1023..95ddc34deb 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/DateJsonDeserializer.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/DateJsonDeserializer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.submarine.websocket; +package org.apache.submarine.server.workbench.websocket; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/Message.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/Message.java similarity index 97% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/Message.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/Message.java index 93ade25db3..b50c8b3c0c 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/Message.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/Message.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.submarine.websocket; +package org.apache.submarine.server.workbench.websocket; import com.google.gson.Gson; import org.slf4j.Logger; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookServer.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookServer.java similarity index 98% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookServer.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookServer.java index 6a42a01e33..192896aa1d 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookServer.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookServer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.submarine.websocket; +package org.apache.submarine.server.workbench.websocket; import com.google.gson.Gson; import com.google.gson.GsonBuilder; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookSocket.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookSocket.java similarity index 97% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookSocket.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookSocket.java index dfaf858032..8525915738 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookSocket.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookSocket.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.submarine.websocket; +package org.apache.submarine.server.workbench.websocket; import org.apache.commons.lang.StringUtils; import org.eclipse.jetty.websocket.api.Session; diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookSocketListener.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookSocketListener.java similarity index 94% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookSocketListener.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookSocketListener.java index 0b9f318f1c..c56484c4ed 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookSocketListener.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookSocketListener.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.submarine.websocket; +package org.apache.submarine.server.workbench.websocket; /** * NoteboookSocket listener. diff --git a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookWebSocketCreator.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookWebSocketCreator.java similarity index 96% rename from submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookWebSocketCreator.java rename to submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookWebSocketCreator.java index c834938d1c..1350b87dd4 100644 --- a/submarine-workbench/workbench-server/src/main/java/org/apache/submarine/websocket/NotebookWebSocketCreator.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/workbench/websocket/NotebookWebSocketCreator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.submarine.websocket; +package org.apache.submarine.server.workbench.websocket; import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest; import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; diff --git a/submarine-workbench/workbench-server/src/main/resources/mbgConfiguration.xml b/submarine-server/server-core/src/main/resources/mbgConfiguration.xml similarity index 100% rename from submarine-workbench/workbench-server/src/main/resources/mbgConfiguration.xml rename to submarine-server/server-core/src/main/resources/mbgConfiguration.xml diff --git a/submarine-workbench/workbench-server/src/main/resources/mybatis-config.xml b/submarine-server/server-core/src/main/resources/mybatis-config.xml similarity index 100% rename from submarine-workbench/workbench-server/src/main/resources/mybatis-config.xml rename to submarine-server/server-core/src/main/resources/mybatis-config.xml diff --git a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/ProjectFilesMapper.xml b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/ProjectFilesMapper.xml similarity index 88% rename from submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/ProjectFilesMapper.xml rename to submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/ProjectFilesMapper.xml index 33de3c3719..2edffc52aa 100644 --- a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/ProjectFilesMapper.xml +++ b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/ProjectFilesMapper.xml @@ -18,8 +18,8 @@ under the License. --> - - + + @@ -27,7 +27,7 @@ - + @@ -48,7 +48,7 @@ - + select REPLACE(UUID(),"-","") @@ -72,7 +72,7 @@ DELETE FROM sys_department WHERE id = #{id} - + UPDATE sys_department SET `dept_code` = #{deptCode}, `dept_name` = #{deptName}, diff --git a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysDictItemMapper.xml b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysDictItemMapper.xml similarity index 89% rename from submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysDictItemMapper.xml rename to submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysDictItemMapper.xml index d2cf8fa51c..1c10dc1b7a 100644 --- a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysDictItemMapper.xml +++ b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysDictItemMapper.xml @@ -18,9 +18,9 @@ under the License. --> - + - + @@ -35,7 +35,7 @@ AND `item_name` like '%${itemName}%' ORDER BY sort_order - + @@ -44,7 +44,7 @@ - + INSERT INTO sys_dict_item (`id`, `dict_code`, `item_code`, `item_name`, `description`, `sort_order`, `deleted`, @@ -63,7 +63,7 @@ DELETE FROM sys_dict_item WHERE id = #{id} - + UPDATE sys_dict_item SET `dict_code` = #{dictCode}, `item_code` = #{itemCode}, diff --git a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysDictMapper.xml b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysDictMapper.xml similarity index 89% rename from submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysDictMapper.xml rename to submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysDictMapper.xml index 6dd628711c..a4e45ef9f0 100644 --- a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysDictMapper.xml +++ b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysDictMapper.xml @@ -18,9 +18,9 @@ under the License. --> - + - + @@ -35,7 +35,7 @@ AND `dict_name` like '%${dictName}%' ORDER BY id - + @@ -43,7 +43,7 @@ - + INSERT INTO sys_dict(`id`, `dict_code`, `dict_name`, `description`, `deleted`, `type`, @@ -71,7 +71,7 @@ DELETE FROM sys_dict WHERE id = #{id} - + UPDATE sys_dict SET `dict_code`=#{dictCode}, `dict_name`=#{dictName}, diff --git a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysMessageMapper.xml b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysMessageMapper.xml similarity index 89% rename from submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysMessageMapper.xml rename to submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysMessageMapper.xml index 83932e9266..28f3298116 100644 --- a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysMessageMapper.xml +++ b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysMessageMapper.xml @@ -18,8 +18,8 @@ under the License. --> - - + + @@ -27,7 +27,7 @@ - + @@ -35,7 +35,7 @@ - + @@ -60,18 +60,18 @@ where id = #{id,jdbcType=VARCHAR} - - insert into sys_message (id, sender, receiver, - type, status, create_by, + + insert into sys_message (id, sender, receiver, + type, status, create_by, create_time, update_time, context) values (REPLACE(UUID(),"-",""), #{sender,jdbcType=VARCHAR}, #{receiver,jdbcType=VARCHAR}, - #{type,jdbcType=VARCHAR}, #{status,jdbcType=INTEGER}, #{createBy,jdbcType=VARCHAR}, + #{type,jdbcType=VARCHAR}, #{status,jdbcType=INTEGER}, #{createBy,jdbcType=VARCHAR}, now(), now() #{context,jdbcType=LONGVARCHAR}) - + insert into sys_message @@ -139,7 +139,7 @@ - + update sys_message @@ -173,7 +173,7 @@ where id = #{id,jdbcType=VARCHAR} - + update sys_message set sender = #{sender,jdbcType=VARCHAR}, receiver = #{receiver,jdbcType=VARCHAR}, @@ -187,7 +187,7 @@ where id = #{id,jdbcType=VARCHAR} - + update sys_message set sender = #{sender,jdbcType=VARCHAR}, receiver = #{receiver,jdbcType=VARCHAR}, @@ -199,4 +199,4 @@ update_time = #{updateTime,jdbcType=TIMESTAMP} where id = #{id,jdbcType=VARCHAR} - \ No newline at end of file + diff --git a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysUserMapper.xml b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysUserMapper.xml similarity index 91% rename from submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysUserMapper.xml rename to submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysUserMapper.xml index 875b83e13a..e15bc7a27e 100644 --- a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SysUserMapper.xml +++ b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SysUserMapper.xml @@ -18,8 +18,8 @@ under the License. --> - - + + @@ -27,7 +27,7 @@ - + @@ -53,7 +53,7 @@ SELECT * FROM sys_user WHERE user_name = #{username} AND password = #{password} - + select REPLACE(UUID(),"-","") @@ -90,7 +90,7 @@ #{createBy}, now(), now()) - + UPDATE sys_user SET `user_name` = #{userName}, `real_name` = #{realName}, @@ -138,7 +138,7 @@ DELETE FROM sys_user WHERE id = #{id} - + UPDATE sys_user SET password = #{password}, `update_time`=now() WHERE id = #{id} diff --git a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SystemMapper.xml b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SystemMapper.xml similarity index 93% rename from submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SystemMapper.xml rename to submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SystemMapper.xml index 250cb5992f..41a218d2fe 100644 --- a/submarine-workbench/workbench-server/src/main/resources/org/apache/submarine/database/mappers/SystemMapper.xml +++ b/submarine-server/server-core/src/main/resources/org/apache/submarine/database/mappers/SystemMapper.xml @@ -18,7 +18,7 @@ under the License. --> - +