From c39a5a6f5062fc12eea72457346c834573731230 Mon Sep 17 00:00:00 2001 From: Mat V Date: Tue, 20 Nov 2018 23:55:04 +0100 Subject: [PATCH 1/8] Init 1.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bb803170..782bfcd8 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ fr.yodamad.svn2git svn-2-git - 1.4 + 1.5-SNAPSHOT war Svn 2 GitLab From 7c56ae0042903dceb9ecc3f3029bc4858796b1c6 Mon Sep 17 00:00:00 2001 From: Mat V Date: Wed, 21 Nov 2018 16:23:01 +0100 Subject: [PATCH 2/8] Close #21 : Add mappings details on migration details page --- .../svn2git/service/MappingService.java | 12 ++++++++ .../svn2git/web/rest/MigrationResource.java | 19 +++++++++++-- .../migration/migration-detail.component.html | 24 ++++++++++++---- .../entities/migration/migration.service.ts | 8 ++++++ .../webapp/app/shared/shared-common.module.ts | 13 +++++++-- .../summary/summary-mappings.component.html | 22 +++++++++++++++ .../summary/summary-mappings.component.ts | 28 +++++++++++++++++++ src/main/webapp/i18n/en/migrationProcess.json | 1 + 8 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 src/main/webapp/app/shared/summary/summary-mappings.component.html create mode 100644 src/main/webapp/app/shared/summary/summary-mappings.component.ts diff --git a/src/main/java/fr/yodamad/svn2git/service/MappingService.java b/src/main/java/fr/yodamad/svn2git/service/MappingService.java index d6d01b17..0f816c6d 100644 --- a/src/main/java/fr/yodamad/svn2git/service/MappingService.java +++ b/src/main/java/fr/yodamad/svn2git/service/MappingService.java @@ -70,4 +70,16 @@ public void delete(Long id) { log.debug("Request to delete Mapping : {}", id); mappingRepository.deleteById(id); } + + + /** + * Get all the migrationMappings for a given migration. + * + * @return the list of entities + */ + @Transactional(readOnly = true) + public List findAllForMigration(Long migrationId) { + log.debug("Request to get all Mappings for migration {}", migrationId); + return mappingRepository.findAllByMigration(migrationId); + } } diff --git a/src/main/java/fr/yodamad/svn2git/web/rest/MigrationResource.java b/src/main/java/fr/yodamad/svn2git/web/rest/MigrationResource.java index cbde7192..5246b883 100644 --- a/src/main/java/fr/yodamad/svn2git/web/rest/MigrationResource.java +++ b/src/main/java/fr/yodamad/svn2git/web/rest/MigrationResource.java @@ -1,6 +1,7 @@ package fr.yodamad.svn2git.web.rest; import com.codahale.metrics.annotation.Timed; +import fr.yodamad.svn2git.domain.Mapping; import fr.yodamad.svn2git.domain.Migration; import fr.yodamad.svn2git.domain.MigrationHistory; import fr.yodamad.svn2git.domain.enumeration.StatusEnum; @@ -177,10 +178,10 @@ public ResponseEntity getMigration(@PathVariable Long id) { } /** - * GET /migrations/:id : get the "id" migration. + * GET /migrations/:id/histories : get the histories linked to "id" migration. * * @param id the id of the migration to retrieve - * @return the ResponseEntity with status 200 (OK) and with body the migration, or with status 404 (Not Found) + * @return the ResponseEntity with status 200 (OK) and with body the histories array, or with status 404 (Not Found) */ @GetMapping("/migrations/{id}/histories") @Timed @@ -190,6 +191,20 @@ public ResponseEntity> getMigrationHistories(@PathVariabl return new ResponseEntity<>(histories, null, HttpStatus.OK); } + /** + * GET /migrations/:id/mappings : get the mappings linked to "id" migration. + * + * @param id the id of the migration to retrieve + * @return the ResponseEntity with status 200 (OK) and with body the mappings array, or with status 404 (Not Found) + */ + @GetMapping("/migrations/{id}/mappings") + @Timed + public ResponseEntity> getMigrationMappings(@PathVariable Long id) { + log.debug("REST request to get Migration : {}", id); + List mappings = mappingService.findAllForMigration(id); + return new ResponseEntity<>(mappings, null, HttpStatus.OK); + } + /** * DELETE /migrations/:id : delete the "id" migration. * diff --git a/src/main/webapp/app/entities/migration/migration-detail.component.html b/src/main/webapp/app/entities/migration/migration-detail.component.html index 8257b6d6..de747e0e 100644 --- a/src/main/webapp/app/entities/migration/migration-detail.component.html +++ b/src/main/webapp/app/entities/migration/migration-detail.component.html @@ -3,12 +3,24 @@

Migration {{migration.id}}. {{migration.svnProject | uppercase }}

- - - Migration summary - - - + + + + + Migration summary + + + + + + + + Migration mappings + + + + +
diff --git a/src/main/webapp/app/entities/migration/migration.service.ts b/src/main/webapp/app/entities/migration/migration.service.ts index 3ef88d26..b87231e0 100644 --- a/src/main/webapp/app/entities/migration/migration.service.ts +++ b/src/main/webapp/app/entities/migration/migration.service.ts @@ -9,10 +9,12 @@ import { SERVER_API_URL } from 'app/app.constants'; import { createRequestOption } from 'app/shared'; import { IMigration } from 'app/shared/model/migration.model'; import { IMigrationHistory } from 'app/shared/model/migration-history.model'; +import { IMapping } from 'app/shared/model/mapping.model'; type EntityResponseType = HttpResponse; type EntityArrayResponseType = HttpResponse; type HistoryArrayResponseType = HttpResponse; +type MappingArrayResponseType = HttpResponse; @Injectable({ providedIn: 'root' }) export class MigrationService { @@ -46,6 +48,12 @@ export class MigrationService { .pipe(map((res: HistoryArrayResponseType) => res)); } + findMappings(id: number): Observable { + return this.http + .get(`${this.resourceUrl}/${id}/mappings`, { observe: 'response' }) + .pipe(map((res: MappingArrayResponseType) => res)); + } + query(req?: any): Observable { const options = createRequestOption(req); return this.http diff --git a/src/main/webapp/app/shared/shared-common.module.ts b/src/main/webapp/app/shared/shared-common.module.ts index 16d004f6..ef17b591 100644 --- a/src/main/webapp/app/shared/shared-common.module.ts +++ b/src/main/webapp/app/shared/shared-common.module.ts @@ -3,17 +3,26 @@ import { NgModule } from '@angular/core'; import { Svn2GitSharedLibsModule, FindLanguageFromKeyPipe, JhiAlertComponent, JhiAlertErrorComponent } from './'; import { SummaryCardComponent } from 'app/shared/summary/summary-card.component'; import { DetailsCardComponent } from 'app/shared/summary/summary-details.component'; +import { SummaryMappingsComponent } from 'app/shared/summary/summary-mappings.component'; @NgModule({ imports: [Svn2GitSharedLibsModule], - declarations: [FindLanguageFromKeyPipe, JhiAlertComponent, JhiAlertErrorComponent, SummaryCardComponent, DetailsCardComponent], + declarations: [ + FindLanguageFromKeyPipe, + JhiAlertComponent, + JhiAlertErrorComponent, + SummaryCardComponent, + DetailsCardComponent, + SummaryMappingsComponent + ], exports: [ Svn2GitSharedLibsModule, FindLanguageFromKeyPipe, JhiAlertComponent, JhiAlertErrorComponent, SummaryCardComponent, - DetailsCardComponent + DetailsCardComponent, + SummaryMappingsComponent ] }) export class Svn2GitSharedCommonModule {} diff --git a/src/main/webapp/app/shared/summary/summary-mappings.component.html b/src/main/webapp/app/shared/summary/summary-mappings.component.html new file mode 100644 index 00000000..2a2f89ee --- /dev/null +++ b/src/main/webapp/app/shared/summary/summary-mappings.component.html @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + +
SVN directory{{ mapping.svnDirectory }}Regex{{ mapping.regex }}arrow_right_altGit directory{{ mapping.gitDirectory }}
+ diff --git a/src/main/webapp/app/shared/summary/summary-mappings.component.ts b/src/main/webapp/app/shared/summary/summary-mappings.component.ts new file mode 100644 index 00000000..7bc5176e --- /dev/null +++ b/src/main/webapp/app/shared/summary/summary-mappings.component.ts @@ -0,0 +1,28 @@ +import { Component, Input, OnInit } from '@angular/core'; +import { IMigration } from 'app/shared/model/migration.model'; +import { MigrationService } from 'app/entities/migration'; +import { Mapping } from 'app/shared/model/mapping.model'; + +/** + * Migration summary component + */ +@Component({ + selector: 'jhi-summary-mappings', + templateUrl: 'summary-mappings.component.html', + styleUrls: ['summary-card.component.css'] +}) +export class SummaryMappingsComponent implements OnInit { + @Input() migration: IMigration; + mappings: Mapping[] = []; + + displayedColumns: string[] = ['svn', 'icon', 'git']; + + constructor(private _migrationService: MigrationService) {} + + ngOnInit() { + if (this.migration !== undefined && this.migration.id !== undefined) { + console.log('Loading mappings for migration ' + this.migration.id); + this._migrationService.findMappings(this.migration.id).subscribe(res => (this.mappings = res.body)); + } + } +} diff --git a/src/main/webapp/i18n/en/migrationProcess.json b/src/main/webapp/i18n/en/migrationProcess.json index e91393c4..0d9270cc 100644 --- a/src/main/webapp/i18n/en/migrationProcess.json +++ b/src/main/webapp/i18n/en/migrationProcess.json @@ -30,6 +30,7 @@ "only" : "only", "summary" : "Migration summary", "details" : "Migration details", + "mappings" : "Migration mappings", "svn" : { "check" : { "repository" : "Check your SVN repository" From 0a888f255cdc1a53752b7bde0660e0995e833a09 Mon Sep 17 00:00:00 2001 From: Mat V Date: Wed, 21 Nov 2018 18:38:00 +0100 Subject: [PATCH 3/8] Close #20 : Add mappings details on migration summary page --- .../migration-stepper.component.html | 34 +++++++++++++------ .../migration/migration-stepper.component.ts | 2 +- .../summary/summary-mappings.component.html | 4 +-- .../summary/summary-mappings.component.ts | 10 +++--- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/main/webapp/app/migration/migration-stepper.component.html b/src/main/webapp/app/migration/migration-stepper.component.html index b82c32b4..69dbb695 100644 --- a/src/main/webapp/app/migration/migration-stepper.component.html +++ b/src/main/webapp/app/migration/migration-stepper.component.html @@ -180,17 +180,29 @@ Done - - - {{svnFormGroup.controls['svnRepository'].value | uppercase}} - Migration summary - - - - - - - + + + + + {{svnFormGroup.controls['svnRepository'].value | uppercase}} + Migration summary + + + + + + + + + + + + Migration mappings + + + + + diff --git a/src/main/webapp/app/migration/migration-stepper.component.ts b/src/main/webapp/app/migration/migration-stepper.component.ts index 00d975f1..a8756a84 100644 --- a/src/main/webapp/app/migration/migration-stepper.component.ts +++ b/src/main/webapp/app/migration/migration-stepper.component.ts @@ -238,7 +238,7 @@ export class MigrationStepperComponent implements OnInit { // Mappings if (this.selection !== undefined && !this.selection.isEmpty()) { - this.mig.mappings = this.selection.selected; + this.mig.mappings = this.selection.selected.filter(mapping => mapping.gitDirectory !== undefined); } return this.mig; } diff --git a/src/main/webapp/app/shared/summary/summary-mappings.component.html b/src/main/webapp/app/shared/summary/summary-mappings.component.html index 2a2f89ee..abcdc855 100644 --- a/src/main/webapp/app/shared/summary/summary-mappings.component.html +++ b/src/main/webapp/app/shared/summary/summary-mappings.component.html @@ -1,5 +1,5 @@ - - + +
diff --git a/src/main/webapp/app/shared/summary/summary-mappings.component.ts b/src/main/webapp/app/shared/summary/summary-mappings.component.ts index 7bc5176e..db039375 100644 --- a/src/main/webapp/app/shared/summary/summary-mappings.component.ts +++ b/src/main/webapp/app/shared/summary/summary-mappings.component.ts @@ -1,7 +1,6 @@ import { Component, Input, OnInit } from '@angular/core'; import { IMigration } from 'app/shared/model/migration.model'; import { MigrationService } from 'app/entities/migration'; -import { Mapping } from 'app/shared/model/mapping.model'; /** * Migration summary component @@ -13,16 +12,19 @@ import { Mapping } from 'app/shared/model/mapping.model'; }) export class SummaryMappingsComponent implements OnInit { @Input() migration: IMigration; - mappings: Mapping[] = []; displayedColumns: string[] = ['svn', 'icon', 'git']; constructor(private _migrationService: MigrationService) {} ngOnInit() { - if (this.migration !== undefined && this.migration.id !== undefined) { + if ( + this.migration !== undefined && + this.migration.id !== undefined && + (this.migration.mappings === undefined || this.migration.mappings === null || this.migration.mappings.length === 0) + ) { console.log('Loading mappings for migration ' + this.migration.id); - this._migrationService.findMappings(this.migration.id).subscribe(res => (this.mappings = res.body)); + this._migrationService.findMappings(this.migration.id).subscribe(res => (this.migration.mappings = res.body)); } } } From 14a3c1947f345ab75e56f55caa5e1650544de1f4 Mon Sep 17 00:00:00 2001 From: Mat V Date: Wed, 21 Nov 2018 23:14:45 +0100 Subject: [PATCH 4/8] Manage regex in mapping, simple case --- .../svn2git/service/MigrationManager.java | 68 ++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java b/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java index b65d062f..c3e03444 100644 --- a/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java +++ b/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java @@ -23,6 +23,7 @@ import java.io.*; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.time.LocalDateTime; @@ -33,9 +34,12 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static java.lang.String.format; +import static java.nio.file.Files.walk; @Service public class MigrationManager { @@ -374,9 +378,20 @@ private boolean applyMapping(String gitWorkingDir, Migration migration, String b boolean workDone = false; List results = null; if (!CollectionUtils.isEmpty(mappings)) { - results = mappings.stream() - .map(mapping -> mvDirectory(gitWorkingDir, migration, mapping, branch)) + // Extract mappings with regex + List regexMappings = mappings.stream() + .filter(mapping -> !StringUtils.isEmpty(mapping.getRegex()) && !"*".equals(mapping.getRegex())) .collect(Collectors.toList()); + results = regexMappings.stream() + .map(mapping -> mvRegex(gitWorkingDir, migration, mapping, branch)) + .collect(Collectors.toList()); + + // Remove regex mappings + mappings.removeAll(regexMappings); + results.addAll( + mappings.stream() + .map(mapping -> mvDirectory(gitWorkingDir, migration, mapping, branch)) + .collect(Collectors.toList())); workDone = results.contains(StatusEnum.DONE); } @@ -445,6 +460,55 @@ private StatusEnum mvDirectory(String gitWorkingDir, Migration migration, Mappin } } + /** + * Apply git mv + * @param gitWorkingDir Working directory + * @param migration Current migration + * @param mapping Mapping to apply + * @param branch Current branch + */ + private StatusEnum mvRegex(String gitWorkingDir, Migration migration, Mapping mapping, String branch) { + String msg = format("git mv %s %s based on regex %s on %s", mapping.getSvnDirectory(), mapping.getGitDirectory(), mapping.getRegex(), branch); + MigrationHistory history = startStep(migration, StepEnum.GIT_MV, msg); + + String regex = mapping.getRegex(); + if (mapping.getRegex().startsWith("*")) { regex = '.' + mapping.getRegex(); } + + Pattern p = Pattern.compile(regex); + try { + Path fullPath = Paths.get(gitWorkingDir, mapping.getSvnDirectory()); + long result = walk(fullPath) + .map(Path::toString) + .filter(s -> !s.equals(fullPath.toString())) + .map(s -> s.substring(gitWorkingDir.length())) + .map(p::matcher) + .filter(Matcher::find) + .map(matcher -> matcher.group(0)) + .mapToInt(el -> { + try { + Path gitPath = Paths.get(gitWorkingDir, mapping.getGitDirectory()); + if (!Files.exists(gitPath)) { + Files.createDirectory(gitPath); + } + return execCommand(gitWorkingDir, format("git mv %s %s", el, Paths.get(mapping.getGitDirectory(), el).toString())); + } catch (InterruptedException | IOException e) { + return 1; + } + }).sum(); + + if (result > 0) { + endStep(history, StatusEnum.DONE_WITH_WARNINGS, null); + return StatusEnum.DONE_WITH_WARNINGS; + } else { + endStep(history, StatusEnum.DONE, null); + return StatusEnum.DONE; + } + } catch (IOException ioEx) { + endStep(history, StatusEnum.FAILED, ioEx.getMessage()); + return StatusEnum.DONE_WITH_WARNINGS; + } + } + /** * Apply git mv * @param gitWorkingDir Current working directory From 3c7b0ab78f5bf4a975df52b3bc092a16898fb2df Mon Sep 17 00:00:00 2001 From: Mat V Date: Thu, 22 Nov 2018 00:04:22 +0100 Subject: [PATCH 5/8] Close #4 : Manage regex for mappings --- .../svn2git/service/MigrationManager.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java b/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java index c3e03444..ca9739f1 100644 --- a/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java +++ b/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java @@ -54,6 +54,8 @@ public class MigrationManager { private static final String GIT_PUSH = "git push"; /** Stars to hide sensitive data. */ private static final String STARS = "******"; + /** Execution error. */ + private static final int ERROR_CODE = 128; // Configuration @Value("${gitlab.url}") String gitlabUrl; @@ -486,13 +488,19 @@ private StatusEnum mvRegex(String gitWorkingDir, Migration migration, Mapping ma .map(matcher -> matcher.group(0)) .mapToInt(el -> { try { - Path gitPath = Paths.get(gitWorkingDir, mapping.getGitDirectory()); + Path gitPath; + if (new File(el).getParentFile() == null) { + gitPath = Paths.get(gitWorkingDir, mapping.getGitDirectory()); + } else { + gitPath = Paths.get(gitWorkingDir, mapping.getGitDirectory(), new File(el).getParent()); + } + if (!Files.exists(gitPath)) { - Files.createDirectory(gitPath); + Files.createDirectories(gitPath); } return execCommand(gitWorkingDir, format("git mv %s %s", el, Paths.get(mapping.getGitDirectory(), el).toString())); } catch (InterruptedException | IOException e) { - return 1; + return ERROR_CODE; } }).sum(); @@ -525,7 +533,7 @@ private StatusEnum mv(String gitWorkingDir, Migration migration, String svnDir, // git mv int exitCode = execCommand(gitWorkingDir, gitCommand); - if (128 == exitCode) { + if (ERROR_CODE == exitCode) { endStep(history, StatusEnum.IGNORED, null); return StatusEnum.IGNORED; } else { From 599a76c7c2627d091faedb6f348a68cc4fd3edff Mon Sep 17 00:00:00 2001 From: Mat V Date: Fri, 23 Nov 2018 23:42:02 +0100 Subject: [PATCH 6/8] Close #28 : All selectAll options on svn directories selection --- .../migration-stepper.component.html | 32 +++++++-- .../migration/migration-stepper.component.ts | 67 ++++++++++++++----- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/src/main/webapp/app/migration/migration-stepper.component.html b/src/main/webapp/app/migration/migration-stepper.component.html index 69dbb695..30f9c1b5 100644 --- a/src/main/webapp/app/migration/migration-stepper.component.html +++ b/src/main/webapp/app/migration/migration-stepper.component.html @@ -63,8 +63,8 @@ - @@ -74,11 +74,29 @@
  • Choose directories to migrate : - - - {{directory}} - - +
    • SVN directory {{ mapping.svnDirectory }}
    + + + + + + + + + + + +
    Directory{{ directory }} + + + + + +
  • diff --git a/src/main/webapp/app/migration/migration-stepper.component.ts b/src/main/webapp/app/migration/migration-stepper.component.ts index a8756a84..ae939d12 100644 --- a/src/main/webapp/app/migration/migration-stepper.component.ts +++ b/src/main/webapp/app/migration/migration-stepper.component.ts @@ -23,6 +23,7 @@ export class MigrationStepperComponent implements OnInit { cleaningFormGroup: FormGroup; mappingFormGroup: FormGroup; displayedColumns: string[] = ['svn', 'regex', 'git', 'selectMapping']; + svnDisplayedColumns: string[] = ['svnDir', 'selectSvn']; // Controls gitlabUserKO = true; @@ -34,7 +35,6 @@ export class MigrationStepperComponent implements OnInit { // Input for migrations svnDirectories: string[] = null; - selectedSvnDirectories: string[]; selectedExtensions: string[]; migrationStarted = false; fileUnit = 'M'; @@ -42,6 +42,9 @@ export class MigrationStepperComponent implements OnInit { svnUrl: string; gitlabUrl: string; + /// Svn selections + svnSelection: SelectionModel; + /// Mapping selections initialSelection = []; allowMultiSelect = true; @@ -84,6 +87,7 @@ export class MigrationStepperComponent implements OnInit { svnUser: [''], svnPwd: [''] }); + this.svnSelection = new SelectionModel(this.allowMultiSelect, []); this.cleaningFormGroup = this._formBuilder.group({ fileMaxSize: ['', Validators.min(1)] }); @@ -142,16 +146,6 @@ export class MigrationStepperComponent implements OnInit { }, () => (this.checkingSvnRepo = false)); } - /** - * Get selected projects to migrate - * @param values - */ - onSelectedOptionsChange(values: string[]) { - this.selectedSvnDirectories = values; - this.svnRepoKO = this.selectedSvnDirectories.length === 0; - this.useSvnRootFolder = values.length === 0; - } - /** * Get selected extensions to clean * @param values @@ -186,7 +180,7 @@ export class MigrationStepperComponent implements OnInit { const mig = this.initMigration(''); this._migrationService.create(mig).subscribe(res => console.log(res)); } else { - this.selectedSvnDirectories + this.svnSelection.selected .map(dir => this.initMigration(dir)) .forEach(mig => this._migrationService.create(mig).subscribe(res => console.log(res))); } @@ -201,7 +195,7 @@ export class MigrationStepperComponent implements OnInit { if (this.useSvnRootFolder) { project = this.svnFormGroup.controls['svnRepository'].value; } else { - project = this.selectedSvnDirectories.toString(); + project = this.svnSelection.selected.toString(); } } @@ -284,7 +278,48 @@ export class MigrationStepperComponent implements OnInit { // Force recheck this.svnRepoKO = true; this.svnDirectories = []; - this.selectedSvnDirectories = []; + this.svnSelection.clear(); + } + + /** Whether the number of selected elements matches the total number of rows. */ + isAllSvnSelected() { + const numSelected = this.svnSelection.selected.length; + const numRows = this.svnDirectories.length; + return numSelected === numRows; + } + + /** Selects all rows if they are not all selected; otherwise clear selection. */ + masterSvnToggle() { + if (this.isAllSvnSelected()) { + this.svnSelection.clear(); + this.svnRepoKO = true; + } else { + this.svnDirectories.forEach(row => this.svnSelection.select(row)); + this.useSvnRootFolder = false; + this.svnRepoKO = false; + } + } + + /** + * Toggle svn directory selection change + * @param event + * @param directory + */ + svnToggle(event: any, directory: string) { + if (event) { + this.useSvnRootFolder = false; + return this.svnSelection.toggle(directory); + } + return null; + } + + /** + * When check/uncheck svn directory + * @param directory + */ + svnChecked(directory: string) { + this.svnRepoKO = this.svnSelection.selected.length === 0; + return this.svnSelection.isSelected(directory); } /** Add a custom mapping. */ @@ -307,8 +342,10 @@ export class MigrationStepperComponent implements OnInit { }); } + /** Root svn directory use selection change. */ onSelectionChange(event: MatCheckboxChange) { this.useSvnRootFolder = event.checked; - this.selectedSvnDirectories = []; + this.svnSelection.clear(); + this.svnRepoKO = !event.checked; } } From 366c1f0e6084cd6f872865cab2e713569798ef01 Mon Sep 17 00:00:00 2001 From: Mat V Date: Sat, 24 Nov 2018 01:04:20 +0100 Subject: [PATCH 7/8] Large refactoring to split large MigrationManager in several --- .../fr/yodamad/svn2git/domain/WorkUnit.java | 21 + .../fr/yodamad/svn2git/service/Cleaner.java | 81 +++ .../yodamad/svn2git/service/GitManager.java | 434 ++++++++++++ .../svn2git/service/HistoryManager.java | 54 ++ .../svn2git/service/MigrationManager.java | 639 ++---------------- .../service/util/MigrationConstants.java | 19 + .../yodamad/svn2git/service/util/Shell.java | 114 ++++ 7 files changed, 790 insertions(+), 572 deletions(-) create mode 100644 src/main/java/fr/yodamad/svn2git/domain/WorkUnit.java create mode 100644 src/main/java/fr/yodamad/svn2git/service/Cleaner.java create mode 100644 src/main/java/fr/yodamad/svn2git/service/GitManager.java create mode 100644 src/main/java/fr/yodamad/svn2git/service/HistoryManager.java create mode 100644 src/main/java/fr/yodamad/svn2git/service/util/MigrationConstants.java create mode 100644 src/main/java/fr/yodamad/svn2git/service/util/Shell.java diff --git a/src/main/java/fr/yodamad/svn2git/domain/WorkUnit.java b/src/main/java/fr/yodamad/svn2git/domain/WorkUnit.java new file mode 100644 index 00000000..8734b89c --- /dev/null +++ b/src/main/java/fr/yodamad/svn2git/domain/WorkUnit.java @@ -0,0 +1,21 @@ +package fr.yodamad.svn2git.domain; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Work unit + */ +public class WorkUnit { + + public Migration migration; + public String root; + public String directory; + public AtomicBoolean warnings; + + public WorkUnit(Migration migration, String root, String directory, AtomicBoolean warnings) { + this.migration = migration; + this.root = root; + this.directory = directory; + this.warnings = warnings; + } +} diff --git a/src/main/java/fr/yodamad/svn2git/service/Cleaner.java b/src/main/java/fr/yodamad/svn2git/service/Cleaner.java new file mode 100644 index 00000000..594c9bd0 --- /dev/null +++ b/src/main/java/fr/yodamad/svn2git/service/Cleaner.java @@ -0,0 +1,81 @@ +package fr.yodamad.svn2git.service; + +import com.madgag.git.bfg.cli.Main; +import fr.yodamad.svn2git.domain.MigrationHistory; +import fr.yodamad.svn2git.domain.WorkUnit; +import fr.yodamad.svn2git.domain.enumeration.StatusEnum; +import fr.yodamad.svn2git.domain.enumeration.StepEnum; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.Arrays; + +import static fr.yodamad.svn2git.service.util.Shell.execCommand; +import static java.lang.String.format; + +/** + * Cleaning operations + */ +@Service +public class Cleaner { + + // Manager + private final HistoryManager historyMgr; + + public Cleaner(final HistoryManager historyManager) { + this.historyMgr = historyManager; + } + + /** + * Clean files with forbiddene extensions if configured + * @param workUnit + * @return + */ + public boolean cleanForbiddenExtensions(WorkUnit workUnit) { + boolean clean = false; + if (!StringUtils.isEmpty(workUnit.migration.getForbiddenFileExtensions())) { + // 3.1 Clean files based on their extensions + Arrays.stream(workUnit.migration.getForbiddenFileExtensions().split(",")) + .forEach(s -> { + MigrationHistory innerHistory = historyMgr.startStep(workUnit.migration, StepEnum.GIT_CLEANING, format("Remove files with extension : %s", s)); + try { + Main.main(new String[]{"--delete-files", s, "--no-blob-protection", workUnit.directory}); + historyMgr.endStep(innerHistory, StatusEnum.DONE, null); + } catch (Exception exc) { + historyMgr.endStep(innerHistory, StatusEnum.FAILED, exc.getMessage()); + workUnit.warnings.set(true); + } + }); + clean = true; + } + return clean; + } + + /** + * Clean large files if configured + * @param workUnit + * @return + * @throws IOException + * @throws InterruptedException + */ + public boolean cleanLargeFiles(WorkUnit workUnit) throws IOException, InterruptedException { + boolean clean = false; + if (!StringUtils.isEmpty(workUnit.migration.getMaxFileSize()) && Character.isDigit(workUnit.migration.getMaxFileSize().charAt(0))) { + // 3.2 Clean files based on size + MigrationHistory history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_CLEANING, + format("Remove files bigger than %s", workUnit.migration.getMaxFileSize())); + + String gitCommand = "git gc"; + execCommand(workUnit.directory, gitCommand); + + Main.main(new String[]{ + "--strip-blobs-bigger-than", workUnit.migration.getMaxFileSize(), + "--no-blob-protection", workUnit.directory}); + + clean = true; + historyMgr.endStep(history, StatusEnum.DONE, null); + } + return clean; + } +} diff --git a/src/main/java/fr/yodamad/svn2git/service/GitManager.java b/src/main/java/fr/yodamad/svn2git/service/GitManager.java new file mode 100644 index 00000000..d965f312 --- /dev/null +++ b/src/main/java/fr/yodamad/svn2git/service/GitManager.java @@ -0,0 +1,434 @@ +package fr.yodamad.svn2git.service; + +import fr.yodamad.svn2git.domain.Mapping; +import fr.yodamad.svn2git.domain.MigrationHistory; +import fr.yodamad.svn2git.domain.WorkUnit; +import fr.yodamad.svn2git.domain.enumeration.StatusEnum; +import fr.yodamad.svn2git.domain.enumeration.StepEnum; +import fr.yodamad.svn2git.repository.MappingRepository; +import fr.yodamad.svn2git.service.util.MigrationConstants; +import fr.yodamad.svn2git.service.util.Shell; +import net.logstash.logback.encoder.org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static fr.yodamad.svn2git.service.util.MigrationConstants.*; +import static fr.yodamad.svn2git.service.util.Shell.execCommand; +import static java.lang.String.format; +import static java.nio.file.Files.walk; + +/** + * Git operations manager + */ +@Service +public class GitManager { + + private static final Logger LOG = LoggerFactory.getLogger(GitManager.class); + + // Manager & repository + private final HistoryManager historyMgr; + private final MappingRepository mappingRepository; + + public GitManager(final HistoryManager historyManager, + final MappingRepository mappingRepository) { + this.historyMgr = historyManager; + this.mappingRepository = mappingRepository; + } + + /** + * Clone empty git repository + * @param workUnit + * @return + * @throws IOException + * @throws InterruptedException + */ + public String gitClone(WorkUnit workUnit) throws IOException, InterruptedException { + String svn = StringUtils.isEmpty(workUnit.migration.getSvnProject()) ? + workUnit.migration.getSvnGroup() + : workUnit.migration.getSvnProject(); + + String initCommand = format("git clone %s/%s/%s.git %s", + workUnit.migration.getGitlabUrl(), + workUnit.migration.getGitlabGroup(), + svn, + workUnit.migration.getGitlabGroup()); + + MigrationHistory history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_CLONE, initCommand); + + String mkdir = format("mkdir %s", workUnit.directory); + execCommand(System.getProperty(JAVA_IO_TMPDIR), mkdir); + + // 2.1. Clone as mirror empty repository, required for BFG + execCommand(workUnit.root, initCommand); + + historyMgr.endStep(history, StatusEnum.DONE, null); + return svn; + } + + /** + * Git svn clone command to copy svn as git repository + * @param workUnit + * @throws IOException + * @throws InterruptedException + */ + public void gitSvnClone(WorkUnit workUnit) throws IOException, InterruptedException { + String cloneCommand; + String safeCommand; + if (StringUtils.isEmpty(workUnit.migration.getSvnUser())) { + cloneCommand = format("git svn clone --trunk=%s/trunk --branches=%s/branches --tags=%s/tags %s/%s", + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnUrl(), + workUnit.migration.getSvnGroup()); + safeCommand = cloneCommand; + } else { + String escapedPassword = StringEscapeUtils.escapeJava(workUnit.migration.getSvnPassword()); + cloneCommand = format("echo %s | git svn clone --username %s --trunk=%s/trunk --branches=%s/branches --tags=%s/tags %s/%s", + escapedPassword, + workUnit.migration.getSvnUser(), + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnUrl(), + workUnit.migration.getSvnGroup()); + safeCommand = format("echo %s | git svn clone --username %s --trunk=%s/trunk --branches=%s/branches --tags=%s/tags %s/%s", + STARS, + workUnit.migration.getSvnUser(), + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnProject(), + workUnit.migration.getSvnUrl(), + workUnit.migration.getSvnGroup()); + } + + MigrationHistory history = historyMgr.startStep(workUnit.migration, StepEnum.SVN_CHECKOUT, safeCommand); + execCommand(workUnit.root, cloneCommand, safeCommand); + + historyMgr.endStep(history, StatusEnum.DONE, null); + } + + /** + * List SVN branches & tags cloned + * @param directory working directory + * @return + * @throws InterruptedException + * @throws IOException + */ + public static List branchList(String directory) throws InterruptedException, IOException { + String command = "git branch -r"; + ProcessBuilder builder = new ProcessBuilder(); + if (Shell.isWindows) { + builder.command("cmd.exe", "/c", command); + } else { + builder.command("sh", "-c", command); + } + builder.directory(new File(directory)); + + Process p = builder.start(); + final BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())); + + List remotes = new ArrayList<>(); + reader.lines().iterator().forEachRemaining(remotes::add); + + p.waitFor(); + p.destroy(); + + return remotes; + } + + /** + * Apply mappings configured + * @param workUnit + * @param branch Branch to process + */ + public boolean applyMapping(WorkUnit workUnit, String branch) { + List mappings = mappingRepository.findAllByMigration(workUnit.migration.getId()); + boolean workDone = false; + List results = null; + if (!CollectionUtils.isEmpty(mappings)) { + // Extract mappings with regex + List regexMappings = mappings.stream() + .filter(mapping -> !StringUtils.isEmpty(mapping.getRegex()) && !"*".equals(mapping.getRegex())) + .collect(Collectors.toList()); + results = regexMappings.stream() + .map(mapping -> mvRegex(workUnit, mapping, branch)) + .collect(Collectors.toList()); + + // Remove regex mappings + mappings.removeAll(regexMappings); + results.addAll( + mappings.stream() + .map(mapping -> mvDirectory(workUnit, mapping, branch)) + .collect(Collectors.toList())); + workDone = results.contains(StatusEnum.DONE); + } + + if (workDone) { + MigrationHistory history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_PUSH, format("Push moved elements on %s", branch)); + try { + // git commit + String gitCommand = "git add ."; + execCommand(workUnit.directory, gitCommand); + gitCommand = format("git commit -m \"Apply mappings on %s\"", branch); + execCommand(workUnit.directory, gitCommand); + // git push + execCommand(workUnit.directory, MigrationConstants.GIT_PUSH); + + historyMgr.endStep(history, StatusEnum.DONE, null); + } catch (IOException | InterruptedException iEx) { + historyMgr.endStep(history, StatusEnum.FAILED, iEx.getMessage()); + return false; + } + } + + // No mappings, OK + if (results == null) { + return true; + } + // Some errors, WARNING to be set + return results.contains(StatusEnum.DONE_WITH_WARNINGS); + } + + /** + * Apply git mv + * @param workUnit + * @param mapping Mapping to apply + * @param branch Current branch + */ + private StatusEnum mvDirectory(WorkUnit workUnit, Mapping mapping, String branch) { + MigrationHistory history; + String msg = format("git mv %s %s on %s", mapping.getSvnDirectory(), mapping.getGitDirectory(), branch); + try { + if (mapping.getGitDirectory().equals("/") || mapping.getGitDirectory().equals(".")) { + // For root directory, we need to loop for subdirectory + List results = Files.list(Paths.get(workUnit.directory, mapping.getSvnDirectory())) + .map(d -> mv(workUnit, format("%s/%s", mapping.getSvnDirectory(), d.getFileName().toString()), d.getFileName().toString(), branch)) + .collect(Collectors.toList()); + + if (results.isEmpty()) { + history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_MV, msg); + historyMgr.endStep(history, StatusEnum.IGNORED, null); + return StatusEnum.IGNORED; + } + + if (results.contains(StatusEnum.DONE_WITH_WARNINGS)) { + return StatusEnum.DONE_WITH_WARNINGS; + } + return StatusEnum.DONE; + + } else { + return mv(workUnit, mapping.getSvnDirectory(), mapping.getGitDirectory(), branch); + } + } catch (IOException gitEx) { + LOG.debug("Failed to mv directory", gitEx); + history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_MV, msg); + historyMgr.endStep(history, StatusEnum.IGNORED, null); + return StatusEnum.IGNORED; + } + } + + /** + * Apply git mv + * @param workUnit + * @param mapping Mapping to apply + * @param branch Current branch + */ + private StatusEnum mvRegex(WorkUnit workUnit, Mapping mapping, String branch) { + String msg = format("git mv %s %s based on regex %s on %s", mapping.getSvnDirectory(), mapping.getGitDirectory(), mapping.getRegex(), branch); + MigrationHistory history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_MV, msg); + + String regex = mapping.getRegex(); + if (mapping.getRegex().startsWith("*")) { regex = '.' + mapping.getRegex(); } + + Pattern p = Pattern.compile(regex); + try { + Path fullPath = Paths.get(workUnit.directory, mapping.getSvnDirectory()); + long result = walk(fullPath) + .map(Path::toString) + .filter(s -> !s.equals(fullPath.toString())) + .map(s -> s.substring(workUnit.directory.length())) + .map(p::matcher) + .filter(Matcher::find) + .map(matcher -> matcher.group(0)) + .mapToInt(el -> { + try { + Path gitPath; + if (new File(el).getParentFile() == null) { + gitPath = Paths.get(workUnit.directory, mapping.getGitDirectory()); + } else { + gitPath = Paths.get(workUnit.directory, mapping.getGitDirectory(), new File(el).getParent()); + } + + if (!Files.exists(gitPath)) { + Files.createDirectories(gitPath); + } + return execCommand(workUnit.directory, format("git mv %s %s", el, Paths.get(mapping.getGitDirectory(), el).toString())); + } catch (InterruptedException | IOException e) { + return MigrationConstants.ERROR_CODE; + } + }).sum(); + + if (result > 0) { + historyMgr.endStep(history, StatusEnum.DONE_WITH_WARNINGS, null); + return StatusEnum.DONE_WITH_WARNINGS; + } else { + historyMgr.endStep(history, StatusEnum.DONE, null); + return StatusEnum.DONE; + } + } catch (IOException ioEx) { + historyMgr.endStep(history, StatusEnum.FAILED, ioEx.getMessage()); + return StatusEnum.DONE_WITH_WARNINGS; + } + } + + /** + * Apply git mv + * @param workUnit + * @param svnDir Origin SVN element + * @param gitDir Target Git element + * @param branch Current branch + */ + private StatusEnum mv(WorkUnit workUnit, String svnDir, String gitDir, String branch) { + MigrationHistory history = null; + try { + String gitCommand = format("git mv %s %s on %s", svnDir, gitDir, branch); + history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_MV, gitCommand); + // git mv + int exitCode = execCommand(workUnit.directory, gitCommand); + + if (MigrationConstants.ERROR_CODE == exitCode) { + historyMgr.endStep(history, StatusEnum.IGNORED, null); + return StatusEnum.IGNORED; + } else { + historyMgr.endStep(history, StatusEnum.DONE, null); + return StatusEnum.DONE; + } + } catch (IOException | InterruptedException gitEx) { + LOG.error("Failed to mv directory", gitEx); + historyMgr.endStep(history, StatusEnum.FAILED, gitEx.getMessage()); + return StatusEnum.DONE_WITH_WARNINGS; + } + } + + /** + * Manage branches extracted from SVN + * @param workUnit + * @param remotes + */ + public void manageBranches(WorkUnit workUnit, List remotes) { + List gitBranches = remotes.stream() + .map(String::trim) + // Remove tags + .filter(b -> !b.startsWith(ORIGIN_TAGS)) + // Remove master/trunk + .filter(b -> !b.contains(MASTER)) + .filter(b -> !b.contains("trunk")) + .collect(Collectors.toList()); + + gitBranches.forEach(b -> { + final boolean warn = pushBranch(workUnit, b); + workUnit.warnings.set(workUnit.warnings.get() || warn); + } + ); + } + + /** + * Push a branch + * @param workUnit + * @param branch Branch to migrate + */ + public boolean pushBranch(WorkUnit workUnit, String branch) throws RuntimeException { + String branchName = branch.replaceFirst("refs/remotes/origin/", ""); + branchName = branchName.replaceFirst("origin/", ""); + LOG.debug(format("Branch %s", branchName)); + + MigrationHistory history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_PUSH, branchName); + try { + String gitCommand = format("git checkout -b %s %s", branchName, branch); + execCommand(workUnit.directory, gitCommand); + execCommand(workUnit.directory, MigrationConstants.GIT_PUSH); + + historyMgr.endStep(history, StatusEnum.DONE, null); + + return applyMapping(workUnit, branch); + } catch (IOException | InterruptedException iEx) { + LOG.error("Failed to push branch", iEx); + historyMgr.endStep(history, StatusEnum.FAILED, iEx.getMessage()); + return false; + } + } + + /** + * Manage tags extracted from SVN + * @param workUnit + * @param remotes + */ + public void manageTags(WorkUnit workUnit, List remotes) { + List gitTags = remotes.stream() + .map(String::trim) + // Only tags + .filter(b -> b.startsWith(ORIGIN_TAGS)) + // Remove temp tags + .filter(b -> !b.contains("@")) + .collect(Collectors.toList()); + + gitTags.forEach(t -> { + final boolean warn = pushTag(workUnit, t); + workUnit.warnings.set(workUnit.warnings.get() || warn); + } + ); + } + + /** + * Push a tag + * @param workUnit + * @param tag Tag to migrate + */ + public boolean pushTag(WorkUnit workUnit, String tag) { + MigrationHistory history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_PUSH, tag); + try { + String tagName = tag.replaceFirst(ORIGIN_TAGS, ""); + LOG.debug(format("Tag %s", tagName)); + + String gitCommand = format("git checkout -b tmp_tag %s", tag); + execCommand(workUnit.directory, gitCommand); + + gitCommand = "git checkout master"; + execCommand(workUnit.directory, gitCommand); + + gitCommand = format("git tag %s tmp_tag", tagName); + + execCommand(workUnit.directory, gitCommand); + + gitCommand = format("git push -u origin %s", tagName); + execCommand(workUnit.directory, gitCommand); + + gitCommand = "git branch -D tmp_tag"; + execCommand(workUnit.directory, gitCommand); + + historyMgr.endStep(history, StatusEnum.DONE, null); + } catch (IOException | InterruptedException gitEx) { + LOG.error("Failed to push branch", gitEx); + historyMgr.endStep(history, StatusEnum.FAILED, gitEx.getMessage()); + return false; + } + return false; + } +} diff --git a/src/main/java/fr/yodamad/svn2git/service/HistoryManager.java b/src/main/java/fr/yodamad/svn2git/service/HistoryManager.java new file mode 100644 index 00000000..7c1ac1bb --- /dev/null +++ b/src/main/java/fr/yodamad/svn2git/service/HistoryManager.java @@ -0,0 +1,54 @@ +package fr.yodamad.svn2git.service; + +import fr.yodamad.svn2git.domain.Migration; +import fr.yodamad.svn2git.domain.MigrationHistory; +import fr.yodamad.svn2git.domain.enumeration.StatusEnum; +import fr.yodamad.svn2git.domain.enumeration.StepEnum; +import fr.yodamad.svn2git.repository.MigrationHistoryRepository; +import org.springframework.stereotype.Service; + +import java.time.Instant; + +/** + * Migration history operations + */ +@Service +public class HistoryManager { + + private final MigrationHistoryRepository migrationHistoryRepository; + + public HistoryManager(final MigrationHistoryRepository mhr) { + this.migrationHistoryRepository = mhr; + } + + /** + * Create a new history for migration + * @param migration + * @param step + * @param data + * @return + */ + public MigrationHistory startStep(Migration migration, StepEnum step, String data) { + MigrationHistory history = new MigrationHistory() + .step(step) + .migration(migration) + .date(Instant.now()) + .status(StatusEnum.RUNNING); + + if (data != null) { + history.data(data); + } + + return migrationHistoryRepository.save(history); + } + + /** + * Update history + * @param history + */ + public void endStep(MigrationHistory history, StatusEnum status, String data) { + history.setStatus(status); + if (data != null) history.setData(data); + migrationHistoryRepository.save(history); + } +} diff --git a/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java b/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java index ca9739f1..a159d233 100644 --- a/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java +++ b/src/main/java/fr/yodamad/svn2git/service/MigrationManager.java @@ -1,62 +1,39 @@ package fr.yodamad.svn2git.service; -import com.madgag.git.bfg.cli.Main; -import fr.yodamad.svn2git.domain.Mapping; import fr.yodamad.svn2git.domain.Migration; import fr.yodamad.svn2git.domain.MigrationHistory; +import fr.yodamad.svn2git.domain.WorkUnit; import fr.yodamad.svn2git.domain.enumeration.StatusEnum; import fr.yodamad.svn2git.domain.enumeration.StepEnum; -import fr.yodamad.svn2git.repository.MappingRepository; -import fr.yodamad.svn2git.repository.MigrationHistoryRepository; import fr.yodamad.svn2git.repository.MigrationRepository; import fr.yodamad.svn2git.service.util.GitlabAdmin; -import net.logstash.logback.encoder.org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.gitlab4j.api.GitLabApiException; import org.gitlab4j.api.models.Group; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; +import java.io.File; import java.util.List; -import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; +import static fr.yodamad.svn2git.service.util.MigrationConstants.GIT_PUSH; +import static fr.yodamad.svn2git.service.util.MigrationConstants.MASTER; +import static fr.yodamad.svn2git.service.util.Shell.execCommand; +import static fr.yodamad.svn2git.service.util.Shell.gitWorkingDir; +import static fr.yodamad.svn2git.service.util.Shell.workingDir; import static java.lang.String.format; -import static java.nio.file.Files.walk; +/** + * Migration manager processing all steps + */ @Service public class MigrationManager { - /** Default ref origin for tags. */ - private static final String ORIGIN_TAGS = "origin/tags/"; - /** Temp directory. */ - private static final String JAVA_IO_TMPDIR = "java.io.tmpdir"; - /** Default branch. */ - private static final String MASTER = "master"; - /** Git push command. */ - private static final String GIT_PUSH = "git push"; - /** Stars to hide sensitive data. */ - private static final String STARS = "******"; - /** Execution error. */ - private static final int ERROR_CODE = 128; - // Configuration @Value("${gitlab.url}") String gitlabUrl; @Value("${gitlab.svc-account}") String gitlabSvcUser; @@ -65,21 +42,23 @@ public class MigrationManager { /** Gitlab API. */ private GitlabAdmin gitlab; + // Managers + private final HistoryManager historyMgr; + private final GitManager gitManager; + private final Cleaner cleaner; // Repositories private final MigrationRepository migrationRepository; - private final MigrationHistoryRepository migrationHistoryRepository; - private final MappingRepository mappingRepository; - - private static final boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows"); - public MigrationManager(final GitlabAdmin gitlabAdmin, - final MigrationRepository migrationRepository, - final MigrationHistoryRepository migrationHistoryRepository, - final MappingRepository mappingRepository) { + public MigrationManager(final Cleaner cleaner, + final GitlabAdmin gitlabAdmin, + final GitManager gitManager, + final HistoryManager historyManager, + final MigrationRepository migrationRepository) { + this.cleaner = cleaner; this.gitlab = gitlabAdmin; + this.gitManager = gitManager; + this.historyMgr = historyManager; this.migrationRepository = migrationRepository; - this.migrationHistoryRepository = migrationHistoryRepository; - this.mappingRepository = mappingRepository; } /** @@ -88,11 +67,12 @@ public MigrationManager(final GitlabAdmin gitlabAdmin, */ @Async public void startMigration(final long migrationId) { + String gitCommand; Migration migration = migrationRepository.findById(migrationId).get(); MigrationHistory history = null; - String rootWorkingDir = workingDir(migration); - String gitWorkingDir = gitWorkingDir(migration); - AtomicBoolean withWarnings = new AtomicBoolean(false); + + String rootDir = workingDir(migration); + WorkUnit workUnit = new WorkUnit(migration, rootDir, gitWorkingDir(rootDir, migration.getSvnGroup()), new AtomicBoolean(false)); try { @@ -101,116 +81,25 @@ public void startMigration(final long migrationId) { migrationRepository.save(migration); // 1. Create project on gitlab : OK - history = startStep(migration, StepEnum.GITLAB_PROJECT_CREATION, migration.getGitlabUrl() + migration.getGitlabGroup()); - - GitlabAdmin gitlabAdmin = gitlab; - if (!gitlabUrl.equalsIgnoreCase(migration.getGitlabUrl())) { - gitlabAdmin = new GitlabAdmin(migration.getGitlabUrl(), migration.getGitlabToken()); - } - Group group = gitlabAdmin.groupApi().getGroup(migration.getGitlabGroup()); - - // If no svn project specified, use svn group instead - if (StringUtils.isEmpty(migration.getSvnProject())) { - gitlabAdmin.projectApi().createProject(group.getId(), migration.getSvnGroup()); - } else { - gitlabAdmin.projectApi().createProject(group.getId(), migration.getSvnProject()); - } + createGitlabProject(migration); - endStep(history, StatusEnum.DONE, null); - - // 2. Checkout SVN repository : OK - String svn = StringUtils.isEmpty(migration.getSvnProject()) ? migration.getSvnGroup(): migration.getSvnProject(); - String initCommand = format("git clone %s/%s/%s.git %s", - migration.getGitlabUrl(), - migration.getGitlabGroup(), - svn, - migration.getGitlabGroup()); - - history = startStep(migration, StepEnum.GIT_CLONE, initCommand); - - String mkdir = "mkdir " + gitWorkingDir; - execCommand(System.getProperty(JAVA_IO_TMPDIR), mkdir); - - // 2.1. Clone as mirror empty repository, required for BFG - execCommand(rootWorkingDir, initCommand); - - endStep(history, StatusEnum.DONE, null); + // 2. Checkout empty repository : OK + String svn = gitManager.gitClone(workUnit); // 2.2. SVN checkout - String cloneCommand; - String safeCommand; - if (StringUtils.isEmpty(migration.getSvnUser())) { - cloneCommand = format("git svn clone --trunk=%s/trunk --branches=%s/branches --tags=%s/tags %s/%s", - migration.getSvnProject(), - migration.getSvnProject(), - migration.getSvnProject(), - migration.getSvnUrl(), - migration.getSvnGroup()); - safeCommand = cloneCommand; - } else { - String escapedPassword = StringEscapeUtils.escapeJava(migration.getSvnPassword()); - cloneCommand = format("echo %s | git svn clone --username %s --trunk=%s/trunk --branches=%s/branches --tags=%s/tags %s/%s", - escapedPassword, - migration.getSvnUser(), - migration.getSvnProject(), - migration.getSvnProject(), - migration.getSvnProject(), - migration.getSvnUrl(), - migration.getSvnGroup()); - safeCommand = format("echo %s | git svn clone --username %s --trunk=%s/trunk --branches=%s/branches --tags=%s/tags %s/%s", - STARS, - migration.getSvnUser(), - migration.getSvnProject(), - migration.getSvnProject(), - migration.getSvnProject(), - migration.getSvnUrl(), - migration.getSvnGroup()); - } - - history = startStep(migration, StepEnum.SVN_CHECKOUT, safeCommand); - execCommand(rootWorkingDir, cloneCommand, safeCommand); - - endStep(history, StatusEnum.DONE, null); + gitManager.gitSvnClone(workUnit); // 3. Clean files - boolean clean = false; - String gitCommand; + boolean cleanExtensions = cleaner.cleanForbiddenExtensions(workUnit); + boolean cleanLargeFiles = cleaner.cleanLargeFiles(workUnit); - if (!StringUtils.isEmpty(migration.getForbiddenFileExtensions())) { - // 3.1 Clean files based on their extensions - Arrays.stream(migration.getForbiddenFileExtensions().split(",")) - .forEach(s -> { - MigrationHistory innerHistory = startStep(migration, StepEnum.GIT_CLEANING, format("Remove files with extension : %s", s)); - try { - Main.main(new String[]{"--delete-files", s, "--no-blob-protection", gitWorkingDir}); - endStep(innerHistory, StatusEnum.DONE, null); - } catch (Exception exc) { - endStep(innerHistory, StatusEnum.FAILED, exc.getMessage()); - withWarnings.set(true); - } - }); - clean = true; - } - - if (!StringUtils.isEmpty(migration.getMaxFileSize()) && Character.isDigit(migration.getMaxFileSize().charAt(0))) { - // 3.2 Clean files based on size - history = startStep(migration, StepEnum.GIT_CLEANING, format("Remove files bigger than %s", migration.getMaxFileSize())); - - gitCommand = "git gc"; - execCommand(gitWorkingDir, gitCommand); - - Main.main(new String[]{"--strip-blobs-bigger-than", migration.getMaxFileSize(), "--no-blob-protection", gitWorkingDir}); - clean = true; - endStep(history, StatusEnum.DONE, null); - } - - if (clean) { + if (cleanExtensions || cleanLargeFiles) { gitCommand = "git reflog expire --expire=now --all && git gc --prune=now --aggressive"; - execCommand(gitWorkingDir, gitCommand); + execCommand(workUnit.directory, gitCommand); } // 4. Git push master based on SVN trunk - history = startStep(migration, StepEnum.GIT_PUSH, "SVN trunk -> GitLab master"); + history = historyMgr.startStep(migration, StepEnum.GIT_PUSH, "SVN trunk -> GitLab master"); // if using root, additional step if (StringUtils.isEmpty(migration.getSvnProject())) { @@ -219,67 +108,43 @@ public void startMigration(final long migrationId) { migration.getGitlabUrl(), migration.getGitlabGroup(), svn); - execCommand(gitWorkingDir, remoteCommand); + execCommand(workUnit.directory, remoteCommand); // Push with upstream gitCommand = format("%s --set-upstream origin master", GIT_PUSH); - execCommand(gitWorkingDir, gitCommand); + execCommand(workUnit.directory, gitCommand); } else { - execCommand(gitWorkingDir, GIT_PUSH); + execCommand(workUnit.directory, GIT_PUSH); } - endStep(history, StatusEnum.DONE, null); + historyMgr.endStep(history, StatusEnum.DONE, null); // Clean pending file(s) removed by BFG gitCommand = "git reset --hard origin/master"; - execCommand(gitWorkingDir, gitCommand); + execCommand(workUnit.directory, gitCommand); // 5. Apply mappings if some - boolean warning = applyMapping(gitWorkingDir, migration, MASTER); - withWarnings.set(withWarnings.get() || warning); + boolean warning = gitManager.applyMapping(workUnit, MASTER); + workUnit.warnings.set(workUnit.warnings.get() || warning); // 6. List branches & tags - List remotes = svnList(gitWorkingDir); + List remotes = GitManager.branchList(workUnit.directory); // Extract branches - List gitBranches = remotes.stream() - .map(String::trim) - // Remove tags - .filter(b -> !b.startsWith(ORIGIN_TAGS)) - // Remove master/trunk - .filter(b -> !b.contains(MASTER)) - .filter(b -> !b.contains("trunk")) - .collect(Collectors.toList()); + gitManager.manageBranches(workUnit, remotes); // Extract tags - List gitTags = remotes.stream() - .map(String::trim) - // Only tags - .filter(b -> b.startsWith(ORIGIN_TAGS)) - // Remove temp tags - .filter(b -> !b.contains("@")) - .collect(Collectors.toList()); - - gitBranches.forEach(b -> { - final boolean warn = pushBranch(gitWorkingDir, migration, b); - withWarnings.set(withWarnings.get() || warn); - } - ); - gitTags.forEach(t -> { - final boolean warn = pushTag(gitWorkingDir, migration, t); - withWarnings.set(withWarnings.get() || warn); - } - ); + gitManager.manageTags(workUnit, remotes); // 7. Clean work directory - history = startStep(migration, StepEnum.CLEANING, format("Remove %s", workingDir(migration))); + history = historyMgr.startStep(migration, StepEnum.CLEANING, format("Remove %s", workUnit.root)); try { - FileUtils.deleteDirectory(new File(workingDir(migration))); - endStep(history, StatusEnum.DONE, null); + FileUtils.deleteDirectory(new File(workUnit.root)); + historyMgr.endStep(history, StatusEnum.DONE, null); } catch (Exception exc) { - endStep(history, StatusEnum.FAILED, exc.getMessage()); - withWarnings.set(true); + historyMgr.endStep(history, StatusEnum.FAILED, exc.getMessage()); + workUnit.warnings.set(true); } - if (withWarnings.get()) { + if (workUnit.warnings.get()) { migration.setStatus(StatusEnum.DONE_WITH_WARNINGS); } else{ migration.setStatus(StatusEnum.DONE); @@ -288,7 +153,7 @@ public void startMigration(final long migrationId) { } catch (Exception exc) { if (history != null) { LOG.error("Failed step : " + history.getStep(), exc); - endStep(history, StatusEnum.FAILED, exc.getMessage()); + historyMgr.endStep(history, StatusEnum.FAILED, exc.getMessage()); } migration.setStatus(StatusEnum.FAILED); @@ -297,396 +162,26 @@ public void startMigration(final long migrationId) { } /** - * Get working directory - * @param mig - * @return - */ - private static String workingDir(Migration mig) { - String today = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss")); - if (isWindows) { - return System.getProperty(JAVA_IO_TMPDIR) + "\\" + today + "_" + mig.getId(); - } - return System.getProperty(JAVA_IO_TMPDIR) + "/" + today + "_" + mig.getId(); - } - - /** - * Get git working directory - * @param migration Migration to process - * @return - */ - private static String gitWorkingDir(Migration migration) { - if (isWindows) { - return workingDir(migration) + "\\" + migration.getSvnGroup(); - } - return workingDir(migration) + "/" + migration.getSvnGroup(); - } - - /** - * Execute a commmand through process - * @param directory Directory in which running command - * @param command command to execute - * @throws InterruptedException - * @throws IOException - */ - private static int execCommand(String directory, String command) throws InterruptedException, IOException { - return execCommand(directory, command, command); - } - - /** - * Execute a commmand through process without an alternative command to print in history - * @param directory Directory in which running command - * @param command command to execute - * @param securedCommandToPrint command to print in history without password/token/... - * @throws InterruptedException - * @throws IOException - */ - private static int execCommand(String directory, String command, String securedCommandToPrint) throws InterruptedException, IOException { - ProcessBuilder builder = new ProcessBuilder(); - if (isWindows) { - builder.command("cmd.exe", "/c", command); - } else { - builder.command("sh", "-c", command); - } - - builder.directory(new File(directory)); - - LOG.debug(format("Exec command : %s", securedCommandToPrint)); - LOG.debug(format("in %s", directory)); - - Process process = builder.start(); - StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), LOG::debug); - Executors.newSingleThreadExecutor().submit(streamGobbler); - - StreamGobbler errorStreamGobbler = new StreamGobbler(process.getErrorStream(), LOG::debug); - Executors.newSingleThreadExecutor().submit(errorStreamGobbler); - - int exitCode = process.waitFor(); - LOG.debug(format("Exit : %d", exitCode)); - - assert exitCode == 0; - - return exitCode; - } - - // Tasks - /** - * Apply mappings configured - * @param gitWorkingDir Current working directory - * @param migration Migration in progress - * @param branch Branch to process - */ - private boolean applyMapping(String gitWorkingDir, Migration migration, String branch) { - List mappings = mappingRepository.findAllByMigration(migration.getId()); - boolean workDone = false; - List results = null; - if (!CollectionUtils.isEmpty(mappings)) { - // Extract mappings with regex - List regexMappings = mappings.stream() - .filter(mapping -> !StringUtils.isEmpty(mapping.getRegex()) && !"*".equals(mapping.getRegex())) - .collect(Collectors.toList()); - results = regexMappings.stream() - .map(mapping -> mvRegex(gitWorkingDir, migration, mapping, branch)) - .collect(Collectors.toList()); - - // Remove regex mappings - mappings.removeAll(regexMappings); - results.addAll( - mappings.stream() - .map(mapping -> mvDirectory(gitWorkingDir, migration, mapping, branch)) - .collect(Collectors.toList())); - workDone = results.contains(StatusEnum.DONE); - } - - if (workDone) { - MigrationHistory history = startStep(migration, StepEnum.GIT_PUSH, format("Push moved elements on %s", branch)); - try { - // git commit - String gitCommand = "git add ."; - execCommand(gitWorkingDir, gitCommand); - gitCommand = format("git commit -m \"Apply mappings on %s\"", branch); - execCommand(gitWorkingDir, gitCommand); - // git push - execCommand(gitWorkingDir, GIT_PUSH); - - endStep(history, StatusEnum.DONE, null); - } catch (IOException | InterruptedException iEx) { - endStep(history, StatusEnum.FAILED, iEx.getMessage()); - return false; - } - } - - // No mappings, OK - if (results == null) { - return true; - } - // Some errors, WARNING to be set - return results.contains(StatusEnum.DONE_WITH_WARNINGS); - } - - /** - * Apply git mv - * @param gitWorkingDir Working directory - * @param migration Current migration - * @param mapping Mapping to apply - * @param branch Current branch - */ - private StatusEnum mvDirectory(String gitWorkingDir, Migration migration, Mapping mapping, String branch) { - MigrationHistory history; - String msg = format("git mv %s %s on %s", mapping.getSvnDirectory(), mapping.getGitDirectory(), branch); - try { - if (mapping.getGitDirectory().equals("/") || mapping.getGitDirectory().equals(".")) { - // For root directory, we need to loop for subdirectory - List results = Files.list(Paths.get(gitWorkingDir, mapping.getSvnDirectory())) - .map(d -> mv(gitWorkingDir, migration, format("%s/%s", mapping.getSvnDirectory(), d.getFileName().toString()), d.getFileName().toString(), branch)) - .collect(Collectors.toList()); - - if (results.isEmpty()) { - history = startStep(migration, StepEnum.GIT_MV, msg); - endStep(history, StatusEnum.IGNORED, null); - return StatusEnum.IGNORED; - } - - if (results.contains(StatusEnum.DONE_WITH_WARNINGS)) { - return StatusEnum.DONE_WITH_WARNINGS; - } - return StatusEnum.DONE; - - } else { - return mv(gitWorkingDir, migration, mapping.getSvnDirectory(), mapping.getGitDirectory(), branch); - } - } catch (IOException gitEx) { - LOG.debug("Failed to mv directory", gitEx); - history = startStep(migration, StepEnum.GIT_MV, msg); - endStep(history, StatusEnum.IGNORED, null); - return StatusEnum.IGNORED; - } - } - - /** - * Apply git mv - * @param gitWorkingDir Working directory - * @param migration Current migration - * @param mapping Mapping to apply - * @param branch Current branch - */ - private StatusEnum mvRegex(String gitWorkingDir, Migration migration, Mapping mapping, String branch) { - String msg = format("git mv %s %s based on regex %s on %s", mapping.getSvnDirectory(), mapping.getGitDirectory(), mapping.getRegex(), branch); - MigrationHistory history = startStep(migration, StepEnum.GIT_MV, msg); - - String regex = mapping.getRegex(); - if (mapping.getRegex().startsWith("*")) { regex = '.' + mapping.getRegex(); } - - Pattern p = Pattern.compile(regex); - try { - Path fullPath = Paths.get(gitWorkingDir, mapping.getSvnDirectory()); - long result = walk(fullPath) - .map(Path::toString) - .filter(s -> !s.equals(fullPath.toString())) - .map(s -> s.substring(gitWorkingDir.length())) - .map(p::matcher) - .filter(Matcher::find) - .map(matcher -> matcher.group(0)) - .mapToInt(el -> { - try { - Path gitPath; - if (new File(el).getParentFile() == null) { - gitPath = Paths.get(gitWorkingDir, mapping.getGitDirectory()); - } else { - gitPath = Paths.get(gitWorkingDir, mapping.getGitDirectory(), new File(el).getParent()); - } - - if (!Files.exists(gitPath)) { - Files.createDirectories(gitPath); - } - return execCommand(gitWorkingDir, format("git mv %s %s", el, Paths.get(mapping.getGitDirectory(), el).toString())); - } catch (InterruptedException | IOException e) { - return ERROR_CODE; - } - }).sum(); - - if (result > 0) { - endStep(history, StatusEnum.DONE_WITH_WARNINGS, null); - return StatusEnum.DONE_WITH_WARNINGS; - } else { - endStep(history, StatusEnum.DONE, null); - return StatusEnum.DONE; - } - } catch (IOException ioEx) { - endStep(history, StatusEnum.FAILED, ioEx.getMessage()); - return StatusEnum.DONE_WITH_WARNINGS; - } - } - - /** - * Apply git mv - * @param gitWorkingDir Current working directory - * @param migration Migration in progress - * @param svnDir Origin SVN element - * @param gitDir Target Git element - * @param branch Current branch - */ - private StatusEnum mv(String gitWorkingDir, Migration migration, String svnDir, String gitDir, String branch) { - MigrationHistory history = null; - try { - String gitCommand = format("git mv %s %s on %s", svnDir, gitDir, branch); - history = startStep(migration, StepEnum.GIT_MV, gitCommand); - // git mv - int exitCode = execCommand(gitWorkingDir, gitCommand); - - if (ERROR_CODE == exitCode) { - endStep(history, StatusEnum.IGNORED, null); - return StatusEnum.IGNORED; - } else { - endStep(history, StatusEnum.DONE, null); - return StatusEnum.DONE; - } - } catch (IOException | InterruptedException gitEx) { - LOG.error("Failed to mv directory", gitEx); - endStep(history, StatusEnum.FAILED, gitEx.getMessage()); - return StatusEnum.DONE_WITH_WARNINGS; - } - } - - /** - * List SVN branches & tags cloned - * @param directory working directory - * @return - * @throws InterruptedException - * @throws IOException - */ - private List svnList(String directory) throws InterruptedException, IOException { - String command = "git branch -r"; - ProcessBuilder builder = new ProcessBuilder(); - if (isWindows) { - builder.command("cmd.exe", "/c", command); - } else { - builder.command("sh", "-c", command); - } - builder.directory(new File(directory)); - - Process p = builder.start(); - final BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())); - - List remotes = new ArrayList<>(); - reader.lines().iterator().forEachRemaining(remotes::add); - - p.waitFor(); - p.destroy(); - - return remotes; - } - - /** - * Push a branch - * @param gitWorkingDir Working directory - * @param migration Migration object - * @param branch Branch to migrate - */ - private boolean pushBranch(String gitWorkingDir, Migration migration, String branch) throws RuntimeException { - String branchName = branch.replaceFirst("refs/remotes/origin/", ""); - branchName = branchName.replaceFirst("origin/", ""); - LOG.debug(format("Branch %s", branchName)); - - MigrationHistory history = startStep(migration, StepEnum.GIT_PUSH, branchName); - try { - String gitCommand = format("git checkout -b %s %s", branchName, branch); - execCommand(gitWorkingDir, gitCommand); - execCommand(gitWorkingDir, GIT_PUSH); - - endStep(history, StatusEnum.DONE, null); - - return applyMapping(gitWorkingDir, migration, branch); - } catch (IOException | InterruptedException iEx) { - LOG.error("Failed to push branch", iEx); - endStep(history, StatusEnum.FAILED, iEx.getMessage()); - return false; - } - } - - /** - * Push a tag - * @param gitWorkingDir Current working directory - * @param migration Migration object - * @param tag Tag to migrate - */ - private boolean pushTag(String gitWorkingDir, Migration migration, String tag) { - MigrationHistory history = startStep(migration, StepEnum.GIT_PUSH, tag); - try { - String tagName = tag.replaceFirst(ORIGIN_TAGS, ""); - LOG.debug(format("Tag %s", tagName)); - - String gitCommand = format("git checkout -b tmp_tag %s", tag); - execCommand(gitWorkingDir, gitCommand); - - gitCommand = "git checkout master"; - execCommand(gitWorkingDir, gitCommand); - - gitCommand = format("git tag %s tmp_tag", tagName); - - execCommand(gitWorkingDir, gitCommand); - - gitCommand = format("git push -u origin %s", tagName); - execCommand(gitWorkingDir, gitCommand); - - gitCommand = "git branch -D tmp_tag"; - execCommand(gitWorkingDir, gitCommand); - - endStep(history, StatusEnum.DONE, null); - } catch (IOException | InterruptedException gitEx) { - LOG.error("Failed to push branch", gitEx); - endStep(history, StatusEnum.FAILED, gitEx.getMessage()); - return false; - } - return false; - } - - // History management - /** - * Create a new history for migration + * Create project in GitLab * @param migration - * @param step - * @param data - * @return + * @throws GitLabApiException */ - private MigrationHistory startStep(Migration migration, StepEnum step, String data) { - MigrationHistory history = new MigrationHistory() - .step(step) - .migration(migration) - .date(Instant.now()) - .status(StatusEnum.RUNNING); + private void createGitlabProject(Migration migration) throws GitLabApiException { + MigrationHistory history = historyMgr.startStep(migration, StepEnum.GITLAB_PROJECT_CREATION, migration.getGitlabUrl() + migration.getGitlabGroup()); - if (data != null) { - history.data(data); + GitlabAdmin gitlabAdmin = gitlab; + if (!gitlabUrl.equalsIgnoreCase(migration.getGitlabUrl())) { + gitlabAdmin = new GitlabAdmin(migration.getGitlabUrl(), migration.getGitlabToken()); } + Group group = gitlabAdmin.groupApi().getGroup(migration.getGitlabGroup()); - return migrationHistoryRepository.save(history); - } - - /** - * Update history - * @param history - */ - private void endStep(MigrationHistory history, StatusEnum status, String data) { - history.setStatus(status); - if (data != null) history.setData(data); - migrationHistoryRepository.save(history); - } - - - // Utils - private static class StreamGobbler implements Runnable { - private InputStream inputStream; - private Consumer consumer; - - StreamGobbler(InputStream inputStream, Consumer consumer) { - this.inputStream = inputStream; - this.consumer = consumer; + // If no svn project specified, use svn group instead + if (StringUtils.isEmpty(migration.getSvnProject())) { + gitlabAdmin.projectApi().createProject(group.getId(), migration.getSvnGroup()); + } else { + gitlabAdmin.projectApi().createProject(group.getId(), migration.getSvnProject()); } - @Override - public void run() { - new BufferedReader(new InputStreamReader(inputStream)).lines() - .forEach(consumer); - } + historyMgr.endStep(history, StatusEnum.DONE, null); } } diff --git a/src/main/java/fr/yodamad/svn2git/service/util/MigrationConstants.java b/src/main/java/fr/yodamad/svn2git/service/util/MigrationConstants.java new file mode 100644 index 00000000..bb5be8e0 --- /dev/null +++ b/src/main/java/fr/yodamad/svn2git/service/util/MigrationConstants.java @@ -0,0 +1,19 @@ +package fr.yodamad.svn2git.service.util; + +/** + * Migrations constants + */ +public abstract class MigrationConstants { + /** Default ref origin for tags. */ + public static final String ORIGIN_TAGS = "origin/tags/"; + /** Temp directory. */ + public static final String JAVA_IO_TMPDIR = "java.io.tmpdir"; + /** Default branch. */ + public static final String MASTER = "master"; + /** Git push command. */ + public static final String GIT_PUSH = "git push"; + /** Stars to hide sensitive data. */ + public static final String STARS = "******"; + /** Execution error. */ + public static final int ERROR_CODE = 128; +} diff --git a/src/main/java/fr/yodamad/svn2git/service/util/Shell.java b/src/main/java/fr/yodamad/svn2git/service/util/Shell.java new file mode 100644 index 00000000..8f3ca9d4 --- /dev/null +++ b/src/main/java/fr/yodamad/svn2git/service/util/Shell.java @@ -0,0 +1,114 @@ +package fr.yodamad.svn2git.service.util; + +import fr.yodamad.svn2git.domain.Migration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.concurrent.Executors; +import java.util.function.Consumer; + +import static fr.yodamad.svn2git.service.util.MigrationConstants.JAVA_IO_TMPDIR; +import static java.lang.String.format; + +/** + * Shell utilities + */ +public abstract class Shell { + + private static final Logger LOG = LoggerFactory.getLogger(Shell.class); + + public static final boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows"); + + /** + * Get working directory + * @param mig + * @return + */ + public static String workingDir(Migration mig) { + String today = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss")); + if (isWindows) { + return System.getProperty(JAVA_IO_TMPDIR) + "\\" + today + "_" + mig.getId(); + } + return System.getProperty(JAVA_IO_TMPDIR) + "/" + today + "_" + mig.getId(); + } + + /** + * Get git working directory + * @param root + * @param sub + * @return + */ + public static String gitWorkingDir(String root, String sub) { + if (isWindows) { + return root + "\\" + sub; + } + return root + "/" + sub; + } + + /** + * Execute a commmand through process + * @param directory Directory in which running command + * @param command command to execute + * @throws InterruptedException + * @throws IOException + */ + public static int execCommand(String directory, String command) throws InterruptedException, IOException { + return execCommand(directory, command, command); + } + + /** + * Execute a commmand through process without an alternative command to print in history + * @param directory Directory in which running command + * @param command command to execute + * @param securedCommandToPrint command to print in history without password/token/... + * @throws InterruptedException + * @throws IOException + */ + public static int execCommand(String directory, String command, String securedCommandToPrint) throws InterruptedException, IOException { + ProcessBuilder builder = new ProcessBuilder(); + if (isWindows) { + builder.command("cmd.exe", "/c", command); + } else { + builder.command("sh", "-c", command); + } + + builder.directory(new File(directory)); + + LOG.debug(format("Exec command : %s", securedCommandToPrint)); + LOG.debug(format("in %s", directory)); + + Process process = builder.start(); + StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), LOG::debug); + Executors.newSingleThreadExecutor().submit(streamGobbler); + + StreamGobbler errorStreamGobbler = new StreamGobbler(process.getErrorStream(), LOG::debug); + Executors.newSingleThreadExecutor().submit(errorStreamGobbler); + + int exitCode = process.waitFor(); + LOG.debug(format("Exit : %d", exitCode)); + + assert exitCode == 0; + + return exitCode; + } + + // Utils + private static class StreamGobbler implements Runnable { + private InputStream inputStream; + private Consumer consumer; + + StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } +} From 2972f6da4cbda04dc0523901b1af7d17cfae30a5 Mon Sep 17 00:00:00 2001 From: Mat V Date: Sat, 24 Nov 2018 01:04:44 +0100 Subject: [PATCH 8/8] Prepare release 1.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 782bfcd8..e86f5c0e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ fr.yodamad.svn2git svn-2-git - 1.5-SNAPSHOT + 1.5 war Svn 2 GitLab