From 7b08a3a30b41501ce5b76386fa0705ed1d94049f Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Mon, 7 Sep 2026 17:14:20 +0200 Subject: [PATCH 01/14] Start infrastructure mixed model importer --- .../BitBucketModelImporter.class.st | 15 --- .../GitlabModelImporter.class.st | 38 ++------ .../GitModelImporter.class.st | 45 ++++++--- .../GitLocalModelImporter.class.st | 70 ++++++++++++-- .../GitModelMixedImporter.class.st | 91 +++++++++++++++++++ .../package.st | 1 + 6 files changed, 194 insertions(+), 66 deletions(-) create mode 100644 src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st create mode 100644 src/GitProjectHealth-Model-MixedImporter/package.st diff --git a/src/BitBucketHealth-Model-Importer/BitBucketModelImporter.class.st b/src/BitBucketHealth-Model-Importer/BitBucketModelImporter.class.st index 96e46172..945cecde 100644 --- a/src/BitBucketHealth-Model-Importer/BitBucketModelImporter.class.st +++ b/src/BitBucketHealth-Model-Importer/BitBucketModelImporter.class.st @@ -1,9 +1,6 @@ Class { #name : #BitBucketModelImporter, #superclass : #GitModelImporter, - #instVars : [ - 'withInitialCommits' - ], #category : #'BitBucketHealth-Model-Importer' } @@ -754,15 +751,3 @@ BitBucketModelImporter >> parseUserIntoGLHUser: userDictionnary [ id: (userDictionnary at: #id); username: (userDictionnary at: #name) ] - -{ #category : #accessing } -BitBucketModelImporter >> withInitialCommits [ - - ^ withInitialCommits -] - -{ #category : #accessing } -BitBucketModelImporter >> withInitialCommits: anObject [ - - withInitialCommits := anObject -] diff --git a/src/GitLabHealth-Model-Importer/GitlabModelImporter.class.st b/src/GitLabHealth-Model-Importer/GitlabModelImporter.class.st index da44303f..68b5e2cb 100644 --- a/src/GitLabHealth-Model-Importer/GitlabModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GitlabModelImporter.class.st @@ -1,10 +1,6 @@ Class { #name : #GitlabModelImporter, #superclass : #GitModelImporter, - #instVars : [ - 'withInitialCommits', - 'withInitialMergeRequest' - ], #category : #'GitLabHealth-Model-Importer' } @@ -179,28 +175,26 @@ GitlabModelImporter >> completeImportedProject: aGLHProject [ add: aGLHProject unless: self blockOnIdEquality. - self importLatestPipelinesOfProject: importedProject. + self factoryImporter importLatestPipelinesOfProject: importedProject. "aGLHProject creator: (self importUser: aGLHProject creator_id)." - (self importUser: importedProject creator_id) addCreatedProject: - importedProject. + (self factoryImporter importUser: importedProject creator_id) + addCreatedProject: importedProject. + - importedProject repository: GLHRepository new. self glhModel add: importedProject repository. self completeImportsOfRepository: importedProject repository. - + withInitialMergeRequest ifTrue: [ - self + self factoryImporter importMergeRequestsOfProject: importedProject since: DateAndTime today until: DateAndTime now ]. ^ importedProject - - ] { #category : #'import - repositories' } @@ -591,18 +585,6 @@ GitlabModelImporter >> glhApi: anObject [ repoApi := anObject ] -{ #category : #accessing } -GitlabModelImporter >> glhModel [ - - ^ glhModel -] - -{ #category : #accessing } -GitlabModelImporter >> glhModel: anObject [ - - glhModel := anObject -] - { #category : #'import - users' } GitlabModelImporter >> importActiveHumanUsers [ @@ -1570,7 +1552,7 @@ GitlabModelImporter >> importProject: aProjectID [ | result projectResult | ('Import project with id: ' , aProjectID printString) recordInfo. - (glhModel allWithType: GLHProject) + (self glhModel allWithType: GLHProject) detect: [ :project | project id = aProjectID ] ifFound: [ :project | ^ project ]. @@ -2142,9 +2124,3 @@ GitlabModelImporter >> subGroupsOf: aGroupID [ ^ subgroups ] - -{ #category : #accessing } -GitlabModelImporter >> withInitialCommits: aBoolean [ - - withInitialCommits := aBoolean -] diff --git a/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st b/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st index 8fd7c502..663eaca0 100644 --- a/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st +++ b/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st @@ -16,10 +16,10 @@ Class { 'withCommitDiffs', 'generalReader', 'withCommitDiffsChanges', - 'withDiffRanges' - ], - #classVars : [ - 'currentImporter' + 'withDiffRanges', + 'withInitialCommits', + 'withInitialMergeRequest', + 'factoryImporter' ], #category : #'GitProjectHealth-Model-Importer' } @@ -110,7 +110,7 @@ GitModelImporter class >> initialize [ currentImporter := Dictionary new. ] -{ #category : #initialization } +{ #category : #'class initialization' } GitModelImporter class >> reset [ currentImporter := OrderedDictionary new. ] @@ -148,14 +148,12 @@ GitModelImporter >> blockForBranchEquality [ { #category : #equality } GitModelImporter >> blockForDiffEquality [ -^ [ :existing :new | - | sameSize sameDiff sameFile | - sameSize := existing diffString size = new diffString size. + ^ [ :existing :new | + | sameDiff sameFile | sameDiff := existing diffString = new diffString. sameFile := existing new_path = new new_path. - sameSize and: [ sameDiff and: [ sameFile ] ]. - ] + sameDiff and: [ sameFile ] ] ] { #category : #equality } @@ -407,6 +405,18 @@ GitModelImporter >> configureReaderForUser: reader [ ] +{ #category : #accessing } +GitModelImporter >> factoryImporter [ + + ^ factoryImporter +] + +{ #category : #accessing } +GitModelImporter >> factoryImporter: anObject [ + + factoryImporter := anObject +] + { #category : #utils } GitModelImporter >> filterCommitChanges: aCollection [ @@ -469,7 +479,7 @@ GitModelImporter >> importBranchesOf: aGLHProject [ { #category : #'import - branches' } GitModelImporter >> importBranchesOfProject: aGLHProject [ - ^ self subclassResponsibility + ^ self factoryImporter importBranchesOfRepository: aGLHProject repository ] { #category : #'import - branches' } @@ -740,6 +750,7 @@ GitModelImporter >> initialize [ self withFiles: false. self withDiffRanges: false. self withCommitsSince: (Date today - 1 week) asDateAndTime. + self factoryImporter: self. userCatalogue := GLHUserCatalogueV2 new anImporter: self; yourself. @@ -892,3 +903,15 @@ GitModelImporter >> withFiles: aBoolean [ withFiles := aBoolean ] + +{ #category : #accessing } +GitModelImporter >> withInitialCommits [ + + ^ withInitialCommits +] + +{ #category : #accessing } +GitModelImporter >> withInitialCommits: anObject [ + + withInitialCommits := anObject +] diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index 20287d39..c679f23b 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -87,7 +87,11 @@ GitLocalModelImporter >> createDiffFrom: patch and: delta [ glhDiff new_file: true ]. delta status = LGitDeltaTypeEnum git_delta_deleted ifTrue: [ glhDiff deleted_file: true ]. - [glhDiff diffString: (patch patchString allButFirstLines: 2).] on: Error do: [ self flag: #'some illegal continuation byte for utf-8 encoding can appears here' ]. + [ glhDiff diffString: (patch patchString allButFirstLines: 2) ] + on: Error + do: [ + self flag: + #'some illegal continuation byte for utf-8 encoding can appears here' ]. ^ glhDiff ] @@ -103,12 +107,17 @@ GitLocalModelImporter >> gitRepository [ GitLocalModelImporter >> gitRepositoryOf: aGLHRepository [ ^ aGLHRepository cacheAt: #localImporterGitReference ifAbsentPut: [ - (LGitRepository on: (aGLHRepository - cacheAt: #localImporterReference - ifAbsent: [ - Error signal: 'Local GLHRepository without importer reference' ])) - init; - yourself ] + | val | + val := aGLHRepository + cacheAt: #localImporterReference + ifAbsentPut: [ self tryClone: aGLHRepository project ]. + val ifNil: [ + Error signal: 'Local GLHRepository without importer reference' ]. + + (LGitRepository on: + (aGLHRepository cacheAt: #localImporterReference ifAbsent: [Error signal: 'Local GLHRepository without importer reference'])) + init; + yourself ] ] { #category : #'import - branches' } @@ -143,10 +152,12 @@ GitLocalModelImporter >> importCommit: aLGitCommit [ (self glhModel allWithType: GLHCommit) asOrderedCollection detect: [ :comm | comm id = aLGitCommit name ] ifFound: [ :comm | ^ comm ]. - commit := self glhModel newCommit id: aLGitCommit name; + title: + (aLGitCommit message lines ifEmpty: [ { '' } ]) first; message: aLGitCommit message; + created_at: aLGitCommit time asDateAndTime; committed_date: aLGitCommit time asDateAndTime; committer_email: aLGitCommit author email; committer_name: aLGitCommit author name; @@ -160,10 +171,29 @@ GitLocalModelImporter >> importCommit: aLGitCommit [ ^ commit ] +{ #category : #'import - commits' } +GitLocalModelImporter >> importCommit: aCommitID ofProject: aGLHProject [ + + | gitCommit commit | + gitCommit := LGitCommit + of: (self gitRepositoryOf: aGLHProject repository) + fromHexString: aCommitID. + + commit := self importCommit: gitCommit. + ^ aGLHProject repository commits add: commit keyedBy: [ :el | el id ] +] + { #category : #'import - commits' } GitLocalModelImporter >> importCommitsOfProject: aProject since: sinceDate until: untilDate [ - self importCommitsOfRepository: aProject repository since: sinceDate until: untilDate + | commits | + commits := self + importCommitsOfRepository: aProject repository + since: sinceDate + until: untilDate. + ^ aProject repository commits + addAll: commits + keyedBy: [ :el | el id ] ] { #category : #'import - repositories' } @@ -356,3 +386,25 @@ GitLocalModelImporter >> resolveCreatorOfCommit: aGLHCommit [ ^ aGLHCommit commitCreator ] + +{ #category : #'as yet unclassified' } +GitLocalModelImporter >> tryClone: aGLHProject [ + + | url clonePath gitUrl repo cloneOptions | + aGLHProject ifNil: [ ^ nil ]. + url := aGLHProject web_url ifNil: [ ^ nil ]. + clonePath := (FileSystem workingDirectory / 'msr' / aGLHProject name). + + clonePath exists ifTrue: [ ^ clonePath ]. + clonePath ensureCreateDirectory. + + gitUrl := url , '.git'. + repo := LGitRepository on: clonePath. + + cloneOptions := repo cloneOptionsStructureClass + withCredentialsProvider: + (IceCredentialsProvider defaultForRemoteUrl: url). + + repo clone: gitUrl options: cloneOptions. + ^ clonePath +] diff --git a/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st b/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st new file mode 100644 index 00000000..70399dff --- /dev/null +++ b/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st @@ -0,0 +1,91 @@ +Class { + #name : #GitModelMixedImporter, + #superclass : #GitModelImporter, + #instVars : [ + 'apiImporter', + 'localImporter' + ], + #category : #'GitProjectHealth-Model-MixedImporter' +} + +{ #category : #accessing } +GitModelMixedImporter >> apiImporter [ + + ^ apiImporter +] + +{ #category : #accessing } +GitModelMixedImporter >> apiImporter: anObject [ + + apiImporter := anObject. + apiImporter factoryImporter: self +] + +{ #category : #accessing } +GitModelMixedImporter >> glhModel: anObject [ + + glhModel := anObject. + self apiImporter ifNotNil: [ :importer | importer glhModel: anObject ]. + self localImporter ifNotNil: [ :importer | importer glhModel: anObject ] +] + +{ #category : #'import - branches' } +GitModelMixedImporter >> importBranchesOfProject: aGLHProject [ + + ^ self localImporter importBranchesOfProject: aGLHProject +] + +{ #category : #'import - branches' } +GitModelMixedImporter >> importBranchesOfRepository: aGLHRepository [ + + ^ self localImporter importBranchesOfRepository: aGLHRepository +] + +{ #category : #'import - commits' } +GitModelMixedImporter >> importCommit: aCommitID ofProject: aGLHProject [ + + ^ self localImporter importCommit: aCommitID ofProject: aGLHProject +] + +{ #category : #'import - commits' } +GitModelMixedImporter >> importCommitsOfProject: aGLHProject since: since until: until [ + + ^ self localImporter importCommitsOfProject: aGLHProject since: since until: until +] + +{ #category : #'import - pipelines' } +GitModelMixedImporter >> importLatestPipelinesOfProject: aGLHProject [ + + ^ self apiImporter importLatestPipelinesOfProject: aGLHProject +] + +{ #category : #'import - merge-requests' } +GitModelMixedImporter >> importMergeRequestsOfProject: aGLHProject since: fromDate until: toDate [ + + ^ self apiImporter importMergeRequestsOfProject: aGLHProject since: fromDate until: toDate +] + +{ #category : #'import - projects' } +GitModelMixedImporter >> importProject: id [ + + ^ self apiImporter importProject: id +] + +{ #category : #'import - users' } +GitModelMixedImporter >> importUser: aUserID [ + + ^ self apiImporter importUser: aUserID +] + +{ #category : #accessing } +GitModelMixedImporter >> localImporter [ + + ^ localImporter +] + +{ #category : #accessing } +GitModelMixedImporter >> localImporter: anObject [ + + localImporter := anObject. + localImporter factoryImporter: self +] diff --git a/src/GitProjectHealth-Model-MixedImporter/package.st b/src/GitProjectHealth-Model-MixedImporter/package.st new file mode 100644 index 00000000..fab22fb7 --- /dev/null +++ b/src/GitProjectHealth-Model-MixedImporter/package.st @@ -0,0 +1 @@ +Package { #name : #'GitProjectHealth-Model-MixedImporter' } From a9ca388e482140e7d422ce12ae9023922f0f0479 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Tue, 8 Sep 2026 15:11:02 +0200 Subject: [PATCH 02/14] clone automatically the repo add import redirection fix import commits of project of local importer to return the correct suite of imported tests in the period organize methods --- .../ProjectMetric.class.st | 20 ++--- .../GitlabModelImporter.class.st | 18 +++-- .../GitModelImporter.class.st | 4 +- .../GitLocalModelImporter.class.st | 79 +++++++++++++------ .../LGitRefSpec.extension.st | 18 +++++ .../GitModelMixedImporter.class.st | 40 ++++++++++ 6 files changed, 137 insertions(+), 42 deletions(-) create mode 100644 src/GitProjectHealth-Model-LocalImporter/LGitRefSpec.extension.st diff --git a/src/GitLabHealth-Model-Analysis/ProjectMetric.class.st b/src/GitLabHealth-Model-Analysis/ProjectMetric.class.st index d7367f21..f627e77a 100644 --- a/src/GitLabHealth-Model-Analysis/ProjectMetric.class.st +++ b/src/GitLabHealth-Model-Analysis/ProjectMetric.class.st @@ -52,15 +52,13 @@ ProjectMetric >> loadCommitsSince: since until: until [ until: until. "download commits unless project cache is not empty" - allCommits := project repository - cacheAt: cacheSymbol - ifAbsentPut: [ - | foundCommits | - foundCommits := glhImporter - importCommitsOfProject: project - since: since - until: until. - foundCommits ]. + allCommits := project repository cacheAt: cacheSymbol ifAbsentPut: [ + | foundCommits | + foundCommits := glhImporter + importCommitsOfProject: project + since: since + until: until. + foundCommits ]. allCommits do: [ :commit | glhImporter importCreatorOfCommit: commit ]. glhImporter chainsCommitsFrom: allCommits. @@ -103,7 +101,9 @@ ProjectMetric >> loadMergeRequestsSince: since until: until [ ProjectMetric >> loadProjectCompleteMergeRequestsSince: since until: until [ | mergeRequests | - mergeRequests := self loadProjectMergeRequestsSince: since until: until. + mergeRequests := self + loadProjectMergeRequestsSince: since + until: until. mergeRequests do: [ :mergeRequest | glhImporter importMergeResquestMerger: mergeRequest. diff --git a/src/GitLabHealth-Model-Importer/GitlabModelImporter.class.st b/src/GitLabHealth-Model-Importer/GitlabModelImporter.class.st index 68b5e2cb..0f3766d4 100644 --- a/src/GitLabHealth-Model-Importer/GitlabModelImporter.class.st +++ b/src/GitLabHealth-Model-Importer/GitlabModelImporter.class.st @@ -1190,16 +1190,22 @@ GitlabModelImporter >> importLatestReleaseOfProject: aGLHProject [ ^ foundRelease. ] -{ #category : #'import - commits' } +{ #category : #'import - merge-requests' } GitlabModelImporter >> importMergeRequestCommits: aGLPHEMergeRequest [ | commits result | aGLPHEMergeRequest commits ifNotNil: [ ^ aGLPHEMergeRequest commits ]. - - result := self repoApi mergeRequests commitsOf: aGLPHEMergeRequest iid inProject: aGLPHEMergeRequest project id. - - commits := (result collect: [ :commitsJson | self parseCommitsResult: commitsJson ]) flattened. - commits := commits collect: [ :commit | self importCommit: commit id ofProject: aGLPHEMergeRequest project ]. + + result := self repoApi mergeRequests + commitsOf: aGLPHEMergeRequest iid + inProject: aGLPHEMergeRequest project id. + + commits := (result collect: [ :commitsJson | + self parseCommitsResult: commitsJson ]) flattened. + commits := commits collect: [ :commit | "Could we use factory.. unsure because sometimes commits of squashed MR are only saved on remote" + self + importCommit: commit id + ofProject: aGLPHEMergeRequest project ]. aGLPHEMergeRequest commits: commits. diff --git a/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st b/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st index 663eaca0..433eeb98 100644 --- a/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st +++ b/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st @@ -192,7 +192,7 @@ GitModelImporter >> blockOnNameEquality [ ^ self blockEqualityOn: #name ] -{ #category : #utils } +{ #category : #utilities } GitModelImporter >> chainsCommitsFrom: commitsCollection [ | dic | @@ -612,6 +612,8 @@ GitModelImporter >> importLatestReleaseOfProject: aGLHProject [ { #category : #'import - merge-requests' } GitModelImporter >> importMergeRequestCommits: mergeRequest [ + + self subclassResponsibility ] { #category : #'import - merge-requests' } diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index c679f23b..ac8f9d8e 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -95,6 +95,31 @@ GitLocalModelImporter >> createDiffFrom: patch and: delta [ ^ glhDiff ] +{ #category : #operation } +GitLocalModelImporter >> fetchMRRemotes: repo [ + + | refspecMR remote | + "But will miss old ones https://stackoverflow.com/a/75139038/5257140" + remote := repo allRemotes detect: [ :remote | + remote remoteName = 'origin' ]. + refspecMR := LGitRefSpec fromString: + '+refs/merge-requests/*/head:refs/remotes/origin/mr/*'. + remote fetchSpec: refspecMR. + remote + fetchWithCredentials: + (IceCredentialsProvider defaultForRemoteUrl: remote url) + refSpec: refspecMR +] + +{ #category : #accessing } +GitLocalModelImporter >> gitCommitOf: aGLHCommit [ + + ^ aGLHCommit cacheAt: #importerOrigin ifAbsentPut: [ + LGitCommit + of: (self gitRepositoryOf: aGLHCommit repository) + fromHexString: aGLHCommit id ] +] + { #category : #'import - commits' } GitLocalModelImporter >> gitRepository [ @@ -148,7 +173,7 @@ GitLocalModelImporter >> importBranchesOfRepository: aGLHRepository [ { #category : #'import - commits' } GitLocalModelImporter >> importCommit: aLGitCommit [ - | commit | + | commit gitDiff stats | (self glhModel allWithType: GLHCommit) asOrderedCollection detect: [ :comm | comm id = aLGitCommit name ] ifFound: [ :comm | ^ comm ]. @@ -167,6 +192,16 @@ GitLocalModelImporter >> importCommit: aLGitCommit [ parent_ids: (aLGitCommit parents collect: [ :parent | parent name ]); yourself. + gitDiff := LGitDiff of: aLGitCommit repository. + gitDiff + diffTree: (LGitCommit + of: aLGitCommit repository + fromHexString: commit parent_ids anyOne) tree + toTree: aLGitCommit tree. + stats := LGitDiffStats of: aLGitCommit repository. + gitDiff diff_get_stats: stats diff: gitDiff. + commit additions: (stats diff_stats_insertions: stats). + commit deletions: (stats diff_stats_deletions: stats). self resolveCreatorOfCommit: commit. ^ commit ] @@ -209,7 +244,7 @@ GitLocalModelImporter >> importCommitsOfRepository: aGLHRepository [ GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceDate until: untilDate [ | revwalk commits alreadyExploredCommit | - commits := OrderedCollection new. + commits := Set new. "to stop when already explored" alreadyExploredCommit := aGPHRepository commits collect: [ :commit | commit id ] @@ -217,21 +252,21 @@ GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceD (self gitRepositoryOf: aGPHRepository) allBranches select: #isLocal thenDo: [ :branch | - | gphBranch shouldContinue lastBranchCommits | + | gphBranch shouldContinue currentBranchCommit | gphBranch := (glhModel allWithType: GLHBranch) detect: [ :gBranch | branch gitName = gBranch name ] ifNone: [ nil ]. revwalk := LGitRevwalk of: (self gitRepositoryOf: aGPHRepository). revwalk pushReference: branch. - lastBranchCommits := nil. + currentBranchCommit := nil. shouldContinue := true. [ | glhCommit gitCommit | gitCommit := revwalk next. glhCommit := self importCommit: gitCommit. - lastBranchCommits := glhCommit. + currentBranchCommit := glhCommit. commits add: glhCommit. alreadyExploredCommit add: glhCommit id. glhCommit cacheAt: #importerOrigin put: gitCommit. @@ -239,8 +274,8 @@ GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceD gphBranch commits ifEmpty: [ gphBranch addCommit: glhCommit ] ] doWhileTrue: [ revwalk hasNext and: [ - lastBranchCommits committed_date > sinceDate and: [ - (lastBranchCommits parent_ids anySatisfy: [ :id | + currentBranchCommit committed_date > sinceDate and: [ + (currentBranchCommit parent_ids anySatisfy: [ :id | alreadyExploredCommit includes: id ]) not ] ] ]. self chainsCommitsFrom: commits ]. @@ -252,30 +287,26 @@ GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceD self withFiles ifTrue: [ self connectDiffs: diffs inside: aGPHRepository ] ]. - - ^ commits + ^ (self glhModel allWithType: GLHCommit) select: [ :commit | commit committed_date between: sinceDate and: untilDate ] ] { #category : #'import - diffs' } GitLocalModelImporter >> importDiffOfCommit: commit [ | sourceCommit parentCommit gitDiff | - sourceCommit := commit - cacheAt: #importerOrigin - ifAbsent: [ Error signal: 'missing importer origin' ]. + sourceCommit := self gitCommitOf: commit. gitDiff := LGitDiff of: sourceCommit repository. commit parentCommits ifNotEmpty: [ - parentCommit := commit parentCommits anyOne - cacheAt: #importerOrigin - ifAbsent: [ - Error signal: 'missing importer origin' ]. + parentCommit := self gitCommitOf: commit parentCommits anyOne. gitDiff := gitDiff diffTree: parentCommit tree toTree: sourceCommit tree ] ifEmpty: [ - gitDiff := gitDiff diffTree: FFIExternalObject null toTree: sourceCommit tree ]. + gitDiff := gitDiff + diffTree: FFIExternalObject null + toTree: sourceCommit tree ]. gitDiff diffFindSimilar. commit diffs: ((1 to: gitDiff numberOfDeltas) collect: [ :idx | @@ -283,6 +314,7 @@ GitLocalModelImporter >> importDiffOfCommit: commit [ delta := gitDiff deltas at: idx. patch := gitDiff patchAt: idx. self createDiffFrom: patch and: delta ]). + "changes are added into the model during the import" self withDiffRanges ifTrue: [ commit diffs do: [ :diff | @@ -332,7 +364,7 @@ GitLocalModelImporter >> importLatestCommitsOfProject [ GitLocalModelImporter >> importLatestCommitsOfProject: aGLHProject [ "this importer load based on inner state local git repository" - self importLatestCommitsOfProject + ^ self importLatestCommitsOfProject ] { #category : #'import - repositories' } @@ -387,24 +419,21 @@ GitLocalModelImporter >> resolveCreatorOfCommit: aGLHCommit [ ^ aGLHCommit commitCreator ] -{ #category : #'as yet unclassified' } +{ #category : #operation } GitLocalModelImporter >> tryClone: aGLHProject [ | url clonePath gitUrl repo cloneOptions | aGLHProject ifNil: [ ^ nil ]. url := aGLHProject web_url ifNil: [ ^ nil ]. - clonePath := (FileSystem workingDirectory / 'msr' / aGLHProject name). - + clonePath := FileSystem workingDirectory / 'msr' / aGLHProject name. clonePath exists ifTrue: [ ^ clonePath ]. - clonePath ensureCreateDirectory. - + clonePath ensureCreateDirectory. gitUrl := url , '.git'. repo := LGitRepository on: clonePath. - cloneOptions := repo cloneOptionsStructureClass withCredentialsProvider: (IceCredentialsProvider defaultForRemoteUrl: url). - repo clone: gitUrl options: cloneOptions. + self fetchMRRemotes: repo. ^ clonePath ] diff --git a/src/GitProjectHealth-Model-LocalImporter/LGitRefSpec.extension.st b/src/GitProjectHealth-Model-LocalImporter/LGitRefSpec.extension.st new file mode 100644 index 00000000..ae56ccc0 --- /dev/null +++ b/src/GitProjectHealth-Model-LocalImporter/LGitRefSpec.extension.st @@ -0,0 +1,18 @@ +Extension { #name : #LGitRefSpec } + +{ #category : #'*GitProjectHealth-Model-LocalImporter' } +LGitRefSpec >> initializeFromString: aString [ + + | stream | + + "self assert: ('(\+?\w(\w|\-|\.|/)*)?\:\w(\w|\-|\.|/)*' asRegex matches: aString)." + "this is wrong, for example ''+refs/merge-requests/*/head:refs/remotes/origin/mr/*' should work" + + stream := aString readStream. + [ + (stream peek = $+) ifTrue: [ + self beForced. + stream next ]. + self source: (stream upTo: $:). + self destination: stream upToEnd ] ensure: [ stream close ] +] diff --git a/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st b/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st index 70399dff..c7e55140 100644 --- a/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st +++ b/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st @@ -53,12 +53,30 @@ GitModelMixedImporter >> importCommitsOfProject: aGLHProject since: since until: ^ self localImporter importCommitsOfProject: aGLHProject since: since until: until ] +{ #category : #'import - diffs' } +GitModelMixedImporter >> importDiffOfCommit: aCommit [ + + ^ self localImporter importDiffOfCommit: aCommit +] + { #category : #'import - pipelines' } GitModelMixedImporter >> importLatestPipelinesOfProject: aGLHProject [ ^ self apiImporter importLatestPipelinesOfProject: aGLHProject ] +{ #category : #'import - merge-requests' } +GitModelMixedImporter >> importMergeRequestCommits: mergeRequest [ + + ^ self apiImporter importMergeRequestCommits: mergeRequest +] + +{ #category : #'import - merge-requests' } +GitModelMixedImporter >> importMergeRequestMergeCommits: aGLHMergeRequest [ + + ^ self apiImporter importMergeRequestMergeCommits: aGLHMergeRequest +] + { #category : #'import - merge-requests' } GitModelMixedImporter >> importMergeRequestsOfProject: aGLHProject since: fromDate until: toDate [ @@ -77,6 +95,12 @@ GitModelMixedImporter >> importUser: aUserID [ ^ self apiImporter importUser: aUserID ] +{ #category : #'import - users' } +GitModelMixedImporter >> importUserByUsername: username [ + + ^ self apiImporter importUserByUsername: username +] + { #category : #accessing } GitModelMixedImporter >> localImporter [ @@ -89,3 +113,19 @@ GitModelMixedImporter >> localImporter: anObject [ localImporter := anObject. localImporter factoryImporter: self ] + +{ #category : #accessing } +GitModelMixedImporter >> withCommitDiffs: aBoolean [ + + super withCommitDiffs: aBoolean. + self localImporter withCommitDiffs: aBoolean. + self apiImporter withCommitDiffs: aBoolean. +] + +{ #category : #accessing } +GitModelMixedImporter >> withDiffRanges: aBoolean [ + + super withDiffRanges: aBoolean. + self localImporter ifNotNil: [ :importer | importer withDiffRanges: aBoolean ]. + self apiImporter ifNotNil: [ :importer | importer withDiffRanges: aBoolean ]. +] From 02f12a9b3f54f65c67df12637f437eed3fd37520 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Wed, 9 Sep 2026 08:50:00 +0200 Subject: [PATCH 03/14] work on tags, diffs, notes, import commit selection --- .../Metric.class.st | 3 +- .../GitLocalModelImporter.class.st | 37 ++++++++++--------- .../GitModelMixedImporter.class.st | 16 +++++++- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/GitLabHealth-Model-Analysis/Metric.class.st b/src/GitLabHealth-Model-Analysis/Metric.class.st index 2ce23f7f..b05182ff 100644 --- a/src/GitLabHealth-Model-Analysis/Metric.class.st +++ b/src/GitLabHealth-Model-Analysis/Metric.class.st @@ -51,8 +51,7 @@ Metric >> foundSuccessorOf: commits andCompleteImportForMax: commitLimit [ visitChildCommits: c childCommits toStoreThemIn: analyzingCommits upto: commitLimit. - analyzingCommits add: c. - ]. + analyzingCommits add: c ]. i := 0. size := analyzingCommits size. diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index ac8f9d8e..6174f3f0 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -173,7 +173,7 @@ GitLocalModelImporter >> importBranchesOfRepository: aGLHRepository [ { #category : #'import - commits' } GitLocalModelImporter >> importCommit: aLGitCommit [ - | commit gitDiff stats | + | commit gitDiff stats parentTree | (self glhModel allWithType: GLHCommit) asOrderedCollection detect: [ :comm | comm id = aLGitCommit name ] ifFound: [ :comm | ^ comm ]. @@ -193,11 +193,13 @@ GitLocalModelImporter >> importCommit: aLGitCommit [ (aLGitCommit parents collect: [ :parent | parent name ]); yourself. gitDiff := LGitDiff of: aLGitCommit repository. - gitDiff - diffTree: (LGitCommit - of: aLGitCommit repository - fromHexString: commit parent_ids anyOne) tree - toTree: aLGitCommit tree. + parentTree := commit parent_ids + ifNotEmpty: [ :parentIds | + (LGitCommit + of: aLGitCommit repository + fromHexString: parentIds anyOne) tree ] + ifEmpty: [ FFIExternalObject null ]. + gitDiff diffTree: parentTree toTree: aLGitCommit tree. stats := LGitDiffStats of: aLGitCommit repository. gitDiff diff_get_stats: stats diff: gitDiff. commit additions: (stats diff_stats_insertions: stats). @@ -221,14 +223,10 @@ GitLocalModelImporter >> importCommit: aCommitID ofProject: aGLHProject [ { #category : #'import - commits' } GitLocalModelImporter >> importCommitsOfProject: aProject since: sinceDate until: untilDate [ - | commits | - commits := self - importCommitsOfRepository: aProject repository - since: sinceDate - until: untilDate. - ^ aProject repository commits - addAll: commits - keyedBy: [ :el | el id ] + ^ self + importCommitsOfRepository: aProject repository + since: sinceDate + until: untilDate ] { #category : #'import - repositories' } @@ -243,7 +241,7 @@ GitLocalModelImporter >> importCommitsOfRepository: aGLHRepository [ { #category : #'import - commits' } GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceDate until: untilDate [ - | revwalk commits alreadyExploredCommit | + | revwalk commits alreadyExploredCommit toReturn | commits := Set new. "to stop when already explored" alreadyExploredCommit := aGPHRepository commits @@ -286,8 +284,13 @@ GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceD self importDiffOfCommit: commit ]. self withFiles ifTrue: [ self connectDiffs: diffs inside: aGPHRepository ] ]. - - ^ (self glhModel allWithType: GLHCommit) select: [ :commit | commit committed_date between: sinceDate and: untilDate ] + aGPHRepository commits addAll: commits keyedBy: [ :el | el id ]. + toReturn := self glhModel allWithType: GLHCommit. + sinceDate ifNotNil: [ + toReturn := toReturn select: [ :c | c committed_date > sinceDate ] ]. + untilDate ifNotNil: [ + toReturn := toReturn select: [ :c | c committed_date < untilDate ] ]. + ^ toReturn ] { #category : #'import - diffs' } diff --git a/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st b/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st index c7e55140..0cedc89d 100644 --- a/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st +++ b/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st @@ -56,7 +56,9 @@ GitModelMixedImporter >> importCommitsOfProject: aGLHProject since: since until: { #category : #'import - diffs' } GitModelMixedImporter >> importDiffOfCommit: aCommit [ - ^ self localImporter importDiffOfCommit: aCommit + ^ [ self localImporter importDiffOfCommit: aCommit ] + on: LGit_GIT_ENOTFOUND + do: [ self apiImporter importDiffOfCommit: aCommit ] ] { #category : #'import - pipelines' } @@ -83,12 +85,24 @@ GitModelMixedImporter >> importMergeRequestsOfProject: aGLHProject since: fromDa ^ self apiImporter importMergeRequestsOfProject: aGLHProject since: fromDate until: toDate ] +{ #category : #'import - notes' } +GitModelMixedImporter >> importNotesOfMergeRequest: mergeRequest [ + + ^ self apiImporter importNotesOfMergeRequest: mergeRequest +] + { #category : #'import - projects' } GitModelMixedImporter >> importProject: id [ ^ self apiImporter importProject: id ] +{ #category : #'import - tags' } +GitModelMixedImporter >> importTagsForProject: aGLHProject [ + + ^ self apiImporter importTagsForProject: aGLHProject +] + { #category : #'import - users' } GitModelMixedImporter >> importUser: aUserID [ From b789a131866a20146f1384097aff1e0b2b7e3fd8 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Wed, 9 Sep 2026 09:17:48 +0200 Subject: [PATCH 04/14] we will need to update the metamodel --- src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st index 8c3fb642..02aba235 100644 --- a/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st +++ b/src/GitLabHealth-Model-Generator/GLHMetamodelGenerator.class.st @@ -685,7 +685,7 @@ GLHMetamodelGenerator >> tagProperties [ { #category : #tags } GLHMetamodelGenerator >> tagRelations [ - (tag property: #commit) - (commit property: #tag). + (tag property: #commit) *- (commit property: #tags). (tag property: #release) - (release property: #tag). (tag property: #repository) *-<> (repository property: #tags). ] From 8859aaa7f8d0a94ce4843487906843f19bfd22d7 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Wed, 9 Sep 2026 09:32:01 +0200 Subject: [PATCH 05/14] a commit can have several tags --- src/GitLabHealth-Model/GLHCommit.class.st | 56 +++++++++++++---------- src/GitLabHealth-Model/GLHIssue.class.st | 2 +- src/GitLabHealth-Model/GLHJob.class.st | 2 +- src/GitLabHealth-Model/GLHTag.class.st | 12 ++--- src/GitLabHealth-Model/GLHUser.class.st | 2 +- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/GitLabHealth-Model/GLHCommit.class.st b/src/GitLabHealth-Model/GLHCommit.class.st index 13d2c100..73e79ac9 100644 --- a/src/GitLabHealth-Model/GLHCommit.class.st +++ b/src/GitLabHealth-Model/GLHCommit.class.st @@ -21,11 +21,11 @@ a commit attached to a repository |---| | `branch` | `GLHCommit` | `commits` | `GLHBranch` | | | `childCommits` | `GLHCommit` | `parentCommits` | `GLHCommit` | | -| `commitedMergeRequest` | `GLHCommit` | `mergeRequestCommit` | `GLHMergeRequest` | | | `commitedMergeRequest` | `GLHCommit` | `mergedCommit` | `GLHMergeRequest` | | +| `commitedMergeRequest` | `GLHCommit` | `mergeRequestCommit` | `GLHMergeRequest` | | | `parentCommits` | `GLHCommit` | `childCommits` | `GLHCommit` | | | `squashedMergeRequest` | `GLHCommit` | `squashCommit` | `GLHMergeRequest` | | -| `tag` | `GLHCommit` | `commit` | `GLHTag` | | +| `tags` | `GLHCommit` | `commit` | `GLHTag` | | ## Properties @@ -58,31 +58,31 @@ Class { #traits : 'FamixTNamedEntity + GLHTRef', #classTraits : 'FamixTNamedEntity classTrait + GLHTRef classTrait', #instVars : [ - '#parent_ids => FMProperty defaultValue: \'OrderedCollection new\'', - '#id => FMProperty', - '#short_id => FMProperty', - '#title => FMProperty', - '#author_name => FMProperty', + '#additions => FMProperty', '#author_email => FMProperty', + '#author_name => FMProperty', '#authored_date => FMProperty', - '#committer_name => FMProperty', - '#committer_email => FMProperty', + '#branch => FMMany type: #GLHBranch opposite: #commits', + '#childCommits => FMMany type: #GLHCommit opposite: #parentCommits', + '#commitCreator => FMOne type: #GLHUser opposite: #commits', + '#commitedMergeRequest => FMOne type: #GLHMergeRequest opposite: #mergedCommit', '#committed_date => FMProperty', + '#committer_email => FMProperty', + '#committer_name => FMProperty', '#created_at => FMProperty', - '#message => FMProperty', - '#web_url => FMProperty', - '#additions => FMProperty', '#deletions => FMProperty', - '#repository => FMOne type: #GLHRepository opposite: #commits', - '#branch => FMMany type: #GLHBranch opposite: #commits', '#diffs => FMMany type: #GLHDiff opposite: #commit', - '#commitCreator => FMOne type: #GLHUser opposite: #commits', - '#parentCommits => FMMany type: #GLHCommit opposite: #childCommits', - '#childCommits => FMMany type: #GLHCommit opposite: #parentCommits', + '#id => FMProperty', '#jobs => FMMany type: #GLHJob opposite: #commit', - '#tag => FMOne type: #GLHTag opposite: #commit', - '#commitedMergeRequest => FMOne type: #GLHMergeRequest opposite: #mergedCommit', - '#squashedMergeRequest => FMOne type: #GLHMergeRequest opposite: #squashCommit' + '#message => FMProperty', + '#parentCommits => FMMany type: #GLHCommit opposite: #childCommits', + '#parent_ids => FMProperty defaultValue: \'OrderedCollection new\'', + '#repository => FMOne type: #GLHRepository opposite: #commits', + '#short_id => FMProperty', + '#squashedMergeRequest => FMOne type: #GLHMergeRequest opposite: #squashCommit', + '#tags => FMMany type: #GLHTag opposite: #commit', + '#title => FMProperty', + '#web_url => FMProperty' ], #category : #'GitLabHealth-Model-Entities' } @@ -126,6 +126,12 @@ GLHCommit >> addParentCommit: anObject [ ^ self parentCommits add: anObject ] +{ #category : #adding } +GLHCommit >> addTag: anObject [ + + ^ self tags add: anObject +] + { #category : #accessing } GLHCommit >> additions [ @@ -473,19 +479,19 @@ GLHCommit >> squashedMergeRequest: anObject [ ] { #category : #accessing } -GLHCommit >> tag [ - "Relation named: #tag type: #GLHTag opposite: #commit" +GLHCommit >> tags [ + "Relation named: #tags type: #GLHTag opposite: #commit" - ^ tag + ^ tags ] { #category : #accessing } -GLHCommit >> tag: anObject [ +GLHCommit >> tags: anObject [ - tag := anObject + tags value: anObject ] { #category : #accessing } diff --git a/src/GitLabHealth-Model/GLHIssue.class.st b/src/GitLabHealth-Model/GLHIssue.class.st index 2d6fc7bb..80320c00 100644 --- a/src/GitLabHealth-Model/GLHIssue.class.st +++ b/src/GitLabHealth-Model/GLHIssue.class.st @@ -27,8 +27,8 @@ an Issues help collaboration within a team to plan, track, and deliver work | `description` | `String` | nil | | | `due_date` | `Object` | nil | | | `id` | `Number` | nil | | -| `name` | `String` | nil | Basic name of the entity, not full reference.| | `name` | `String` | nil | | +| `name` | `String` | nil | Basic name of the entity, not full reference.| | `state` | `String` | nil | | | `updated_at` | `Object` | nil | | diff --git a/src/GitLabHealth-Model/GLHJob.class.st b/src/GitLabHealth-Model/GLHJob.class.st index b50f9f3d..bed62ed5 100644 --- a/src/GitLabHealth-Model/GLHJob.class.st +++ b/src/GitLabHealth-Model/GLHJob.class.st @@ -20,8 +20,8 @@ A CI Job | `allow_failure` | `Boolean` | nil | | | `duration` | `Object` | nil | | | `id` | `Number` | nil | | -| `name` | `String` | nil | | | `name` | `String` | nil | Basic name of the entity, not full reference.| +| `name` | `String` | nil | | | `ref` | `String` | nil | | | `web_url` | `String` | nil | | diff --git a/src/GitLabHealth-Model/GLHTag.class.st b/src/GitLabHealth-Model/GLHTag.class.st index 8715d8b0..55558d34 100644 --- a/src/GitLabHealth-Model/GLHTag.class.st +++ b/src/GitLabHealth-Model/GLHTag.class.st @@ -12,7 +12,7 @@ a Tag is a reference to a specific point in the repository's history ### Other | Relation | Origin | Opposite | Type | Comment | |---| -| `commit` | `GLHTag` | `tag` | `GLHCommit` | | +| `commit` | `GLHTag` | `tags` | `GLHCommit` | | | `release` | `GLHTag` | `tag` | `GLHRelease` | | @@ -35,13 +35,13 @@ Class { #traits : 'GLHTRef', #classTraits : 'GLHTRef classTrait', #instVars : [ + '#commit => FMOne type: #GLHCommit opposite: #tags', + '#created_at => FMProperty', '#message => FMProperty', - '#target => FMProperty', '#protected => FMProperty', - '#created_at => FMProperty', - '#commit => FMOne type: #GLHCommit opposite: #tag', '#release => FMOne type: #GLHRelease opposite: #tag', - '#repository => FMOne type: #GLHRepository opposite: #tags' + '#repository => FMOne type: #GLHRepository opposite: #tags', + '#target => FMProperty' ], #category : #'GitLabHealth-Model-Entities' } @@ -57,7 +57,7 @@ GLHTag class >> annotation [ { #category : #accessing } GLHTag >> commit [ - "Relation named: #commit type: #GLHCommit opposite: #tag" + "Relation named: #commit type: #GLHCommit opposite: #tags" ^ commit diff --git a/src/GitLabHealth-Model/GLHUser.class.st b/src/GitLabHealth-Model/GLHUser.class.st index 20a41c87..d6648af4 100644 --- a/src/GitLabHealth-Model/GLHUser.class.st +++ b/src/GitLabHealth-Model/GLHUser.class.st @@ -51,8 +51,8 @@ A GitLab User | `job_title` | `String` | nil | | | `linkedin` | `String` | nil | | | `location` | `String` | nil | | -| `name` | `String` | nil | | | `name` | `String` | nil | Basic name of the entity, not full reference.| +| `name` | `String` | nil | | | `organization` | `String` | nil | | | `pronouns` | `String` | nil | | | `public_email` | `String` | nil | | From 38c294f90f9c32a54a95799f357a31012eb0f232 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Wed, 9 Sep 2026 10:03:50 +0200 Subject: [PATCH 06/14] remove deprecated stuff that no one use anymore --- .../GPModelImporter.class.st | 30 ------------------- .../GitModelImporter.class.st | 28 ----------------- 2 files changed, 58 deletions(-) delete mode 100644 src/GitProjectHealth-Model-Importer/GPModelImporter.class.st diff --git a/src/GitProjectHealth-Model-Importer/GPModelImporter.class.st b/src/GitProjectHealth-Model-Importer/GPModelImporter.class.st deleted file mode 100644 index e3ae04ab..00000000 --- a/src/GitProjectHealth-Model-Importer/GPModelImporter.class.st +++ /dev/null @@ -1,30 +0,0 @@ -" -I am a superclass for Model importer. -I am a try of normalization for the different importer. - -I also define some contract so it is easier to switch from one importer to another -" -Class { - #name : #GPModelImporter, - #superclass : #GitModelImporter, - #classVars : [ - 'currentImporter' - ], - #category : #'GitProjectHealth-Model-Importer' -} - -{ #category : #'accessing - global variables' } -GPModelImporter class >> importers [ - ^ currentImporter -] - -{ #category : #testing } -GPModelImporter class >> isDeprecated [ - - ^ true -] - -{ #category : #initialization } -GPModelImporter class >> reset [ - currentImporter := OrderedDictionary new. -] diff --git a/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st b/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st index 433eeb98..5793eb81 100644 --- a/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st +++ b/src/GitProjectHealth-Model-Importer/GitModelImporter.class.st @@ -100,21 +100,6 @@ GitModelImporter class >> forGitlabWithToken: anApiToken andModel: aGitModel [ ^ self forGitlabHostOn: 'https://gitlab.com/api/v4' withToken: anApiToken andModel: aGitModel ] -{ #category : #'accessing - global variables' } -GitModelImporter class >> importers [ - ^ currentImporter -] - -{ #category : #'class initialization' } -GitModelImporter class >> initialize [ - currentImporter := Dictionary new. -] - -{ #category : #'class initialization' } -GitModelImporter class >> reset [ - currentImporter := OrderedDictionary new. -] - { #category : #initialization } GitModelImporter >> beWithFiles [ @@ -760,19 +745,6 @@ GitModelImporter >> initialize [ self initReader ] -{ #category : #initialization } -GitModelImporter >> makeGlobal [ - ^ self makeGlobal: DateAndTime now printString. -] - -{ #category : #initialization } -GitModelImporter >> makeGlobal: aLabel [ - - currentImporter := GithubModelImporter importers ifNil: [ OrderedDictionary new ]. - currentImporter at: aLabel put: self. - ^ currentImporter -] - { #category : #parsing } GitModelImporter >> parseDiffString: aDiff [ "parse diff string to create entities for each lines" From ee9ada485928f3f27fa5fb4a8a52295fcdaa98e9 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Wed, 9 Sep 2026 14:24:57 +0200 Subject: [PATCH 07/14] kill random failing test --- .../GitLocalModelImporter.class.st | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index 6174f3f0..51524202 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -64,7 +64,10 @@ GitLocalModelImporter >> connectDiffs: diffs inside: aGPHRepository [ indexedFiles := (aGPHRepository allToScope: GLHFile) collect: [ :el | el path -> el ] as: Dictionary ]. - diffs do: [ :diff | + "reverse order to have a smart rename history" + (diffs asOrderedCollection sort: [ :diffA :diffB | + diffA commit committed_date > diffB commit committed_date ]) do: [ + :diff | indexedFiles at: diff new_path ifPresent: [ :gphFile | From 4f92918665c24f7740ee7f3911e98091d1198e1b Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Wed, 9 Sep 2026 14:44:49 +0200 Subject: [PATCH 08/14] fix importBranchesOfRepository: to work even with not checkouting all branches --- .../GitLocalModelImporter.class.st | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index 51524202..424773df 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -154,13 +154,14 @@ GitLocalModelImporter >> importBranchesOfRepository: aGLHRepository [ | branches headCommit | headCommit := (self gitRepositoryOf: aGLHRepository) head object. branches := (self gitRepositoryOf: aGLHRepository) allBranches - select: [ :branch | branch isBranch ] - thenCollect: [ :branch | + collect: [ :branch | GLHBranch new name: branch gitName; - sha: branch targetId hexString; + sha: branch annotatedCommit id hexString; yourself ]. - aGLHRepository branches addAll: branches. + aGLHRepository branches + addAll: branches + keyedBy: [ :el | el name ]. self glhModel addAll: branches keyedBy: [ :el | { el name. From 06870015a5d670e8de9fa1072e80b967a47542ff Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Thu, 10 Sep 2026 16:53:09 +0200 Subject: [PATCH 09/14] better retrieve name of remote branches --- .../GitLocalModelImporter.class.st | 12 ++++++------ .../LGitReference.extension.st | 3 ++- .../GitModelMixedImporter.class.st | 6 +++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index 424773df..052eb149 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -151,7 +151,7 @@ GitLocalModelImporter >> gitRepositoryOf: aGLHRepository [ { #category : #'import - branches' } GitLocalModelImporter >> importBranchesOfRepository: aGLHRepository [ - | branches headCommit | + | branches headCommit addedBranches | headCommit := (self gitRepositoryOf: aGLHRepository) head object. branches := (self gitRepositoryOf: aGLHRepository) allBranches collect: [ :branch | @@ -159,19 +159,19 @@ GitLocalModelImporter >> importBranchesOfRepository: aGLHRepository [ name: branch gitName; sha: branch annotatedCommit id hexString; yourself ]. - aGLHRepository branches - addAll: branches - keyedBy: [ :el | el name ]. + addedBranches := aGLHRepository branches + addAll: branches + keyedBy: [ :el | el name ]. self glhModel addAll: branches keyedBy: [ :el | { el name. el repository } ]. self withFiles ifTrue: [ - branches do: [ :branch | self importFilesOfBranch: branch ] ]. + addedBranches do: [ :branch | self importFilesOfBranch: branch ] ]. (self gitRepositoryOf: aGLHRepository) resetHardTo: headCommit. - ^ branches + ^ addedBranches ] { #category : #'import - commits' } diff --git a/src/GitProjectHealth-Model-LocalImporter/LGitReference.extension.st b/src/GitProjectHealth-Model-LocalImporter/LGitReference.extension.st index 40b25a3b..32c0d279 100644 --- a/src/GitProjectHealth-Model-LocalImporter/LGitReference.extension.st +++ b/src/GitProjectHealth-Model-LocalImporter/LGitReference.extension.st @@ -5,5 +5,6 @@ LGitReference >> gitName [ ^ self isLocal ifTrue: [ self name withoutPrefix: 'refs/heads/' ] - ifFalse: [ self name withoutPrefix: 'refs/remotes/' ] + ifFalse: [ "removing the refs/remote and the name of the remote" + (self name withoutPrefix: 'refs/remotes/') copyAfter: $/ ] ] diff --git a/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st b/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st index 0cedc89d..d9b90b52 100644 --- a/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st +++ b/src/GitProjectHealth-Model-MixedImporter/GitModelMixedImporter.class.st @@ -44,7 +44,11 @@ GitModelMixedImporter >> importBranchesOfRepository: aGLHRepository [ { #category : #'import - commits' } GitModelMixedImporter >> importCommit: aCommitID ofProject: aGLHProject [ - ^ self localImporter importCommit: aCommitID ofProject: aGLHProject + ^ [ + self localImporter importCommit: aCommitID ofProject: aGLHProject ] + on: LGit_GIT_ENOTFOUND + do: [ + self apiImporter importCommit: aCommitID ofProject: aGLHProject ] ] { #category : #'import - commits' } From c737c8b19f433adf506f37d2e230d433b1021a85 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Fri, 11 Sep 2026 14:00:28 +0200 Subject: [PATCH 10/14] add helper method to explore branches of commit fix file import that uses old API to retrieve branch name --- src/GitLabHealth-Model-Extension/GLHCommit.extension.st | 9 +++++++++ .../GitLocalModelImporter.class.st | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st index 8406ee13..d4c3c513 100644 --- a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st +++ b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st @@ -1,5 +1,14 @@ Extension { #name : #GLHCommit } +{ #category : #'*GitLabHealth-Model-Extension' } +GLHCommit >> allBranches [ + "A commit is associated to a branch only if it is the head of its branch. This method search all the branches by exploring the children commit " + + ^ (self + deep: [ :commit | commit childCommits ] + flatCollect: [ :commit | commit branch ]) , { self branch } +] + { #category : #'*GitLabHealth-Model-Extension' } GLHCommit >> computeChurnForMaxCommits: aLimit [ diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index 052eb149..e29bc627 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -337,7 +337,7 @@ GitLocalModelImporter >> importFilesOfBranch: aGLHBranch [ | gitBranch | gitRepository := self gitRepositoryOf: aGLHBranch repository. gitBranch := gitRepository allBranches detect: [ :branch | - branch shortName = aGLHBranch name ]. + branch gitName = aGLHBranch name ]. gitRepository resetHardTo: gitBranch object. From 8cf19f310f8f0bf28ad94b40001c1c8515a6e0c5 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Fri, 11 Sep 2026 14:02:43 +0200 Subject: [PATCH 11/14] wrong add method call to collection --- src/GitLabHealth-Model-Extension/GLHCommit.extension.st | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st index d4c3c513..5fdf4387 100644 --- a/src/GitLabHealth-Model-Extension/GLHCommit.extension.st +++ b/src/GitLabHealth-Model-Extension/GLHCommit.extension.st @@ -6,7 +6,7 @@ GLHCommit >> allBranches [ ^ (self deep: [ :commit | commit childCommits ] - flatCollect: [ :commit | commit branch ]) , { self branch } + flatCollect: [ :commit | commit branch ]) , self branch ] { #category : #'*GitLabHealth-Model-Extension' } From 2be49b5ff66a96f5718590e07eae6b62d1cd4f01 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Tue, 29 Sep 2026 11:14:47 +0200 Subject: [PATCH 12/14] do not import already imported information --- .../GitLocalModelImporter.class.st | 103 +++++++++--------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index e29bc627..d1661402 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -251,36 +251,32 @@ GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceD alreadyExploredCommit := aGPHRepository commits collect: [ :commit | commit id ] as: Set. - (self gitRepositoryOf: aGPHRepository) allBranches - select: #isLocal - thenDo: [ :branch | - | gphBranch shouldContinue currentBranchCommit | - gphBranch := (glhModel allWithType: GLHBranch) - detect: [ :gBranch | branch gitName = gBranch name ] - ifNone: [ nil ]. - - revwalk := LGitRevwalk of: (self gitRepositoryOf: aGPHRepository). - revwalk pushReference: branch. - currentBranchCommit := nil. - shouldContinue := true. - [ - | glhCommit gitCommit | - gitCommit := revwalk next. - glhCommit := self importCommit: gitCommit. - - currentBranchCommit := glhCommit. - commits add: glhCommit. - alreadyExploredCommit add: glhCommit id. - glhCommit cacheAt: #importerOrigin put: gitCommit. - - gphBranch commits ifEmpty: [ gphBranch addCommit: glhCommit ] ] - doWhileTrue: [ - revwalk hasNext and: [ - currentBranchCommit committed_date > sinceDate and: [ - (currentBranchCommit parent_ids anySatisfy: [ :id | - alreadyExploredCommit includes: id ]) not ] ] ]. - - self chainsCommitsFrom: commits ]. + (self gitRepositoryOf: aGPHRepository) allBranches do: [ :branch | + | gphBranch shouldContinue currentBranchCommit | + gphBranch := (glhModel allWithType: GLHBranch) + detect: [ :gBranch | branch gitName = gBranch name ] + ifNone: [ nil ]. + + revwalk := LGitRevwalk of: (self gitRepositoryOf: aGPHRepository). + revwalk pushReference: branch. + currentBranchCommit := nil. + shouldContinue := true. + [ + | glhCommit gitCommit | + gitCommit := revwalk next. + glhCommit := self importCommit: gitCommit. + + currentBranchCommit := glhCommit. + commits add: glhCommit. + alreadyExploredCommit add: glhCommit id. + glhCommit cacheAt: #importerOrigin put: gitCommit. + + gphBranch commits ifEmpty: [ gphBranch addCommit: glhCommit ] ] + doWhileTrue: [ + revwalk hasNext and: [ + currentBranchCommit committed_date > sinceDate ] ]. + + self chainsCommitsFrom: commits ]. self withCommitDiffs ifTrue: [ | diffs | @@ -300,27 +296,32 @@ GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceD { #category : #'import - diffs' } GitLocalModelImporter >> importDiffOfCommit: commit [ - | sourceCommit parentCommit gitDiff | - sourceCommit := self gitCommitOf: commit. - gitDiff := LGitDiff of: sourceCommit repository. - - commit parentCommits - ifNotEmpty: [ - parentCommit := self gitCommitOf: commit parentCommits anyOne. - gitDiff := gitDiff - diffTree: parentCommit tree - toTree: sourceCommit tree ] - ifEmpty: [ - gitDiff := gitDiff - diffTree: FFIExternalObject null - toTree: sourceCommit tree ]. - - gitDiff diffFindSimilar. - commit diffs: ((1 to: gitDiff numberOfDeltas) collect: [ :idx | - | delta patch | - delta := gitDiff deltas at: idx. - patch := gitDiff patchAt: idx. - self createDiffFrom: patch and: delta ]). + | sourceCommit gitDiff | + commit diffs ifNotEmpty: [ + sourceCommit := self gitCommitOf: commit. + gitDiff := LGitDiff of: sourceCommit repository. + + commit parent_ids + ifNotEmpty: [ + | parentTree | + parentTree := commit parent_ids + ifNotEmpty: [ :parentIds | + (LGitCommit + of: sourceCommit repository + fromHexString: parentIds anyOne) tree ] + ifEmpty: [ FFIExternalObject null ]. + gitDiff := gitDiff diffTree: parentTree toTree: sourceCommit tree ] + ifEmpty: [ + gitDiff := gitDiff + diffTree: FFIExternalObject null + toTree: sourceCommit tree ]. + + gitDiff diffFindSimilar. + commit diffs: ((1 to: gitDiff numberOfDeltas) collect: [ :idx | + | delta patch | + delta := gitDiff deltas at: idx. + patch := gitDiff patchAt: idx. + self createDiffFrom: patch and: delta ]) ]. "changes are added into the model during the import" self withDiffRanges ifTrue: [ From 164554a0e0f809128a016e7aef6d0df798f15271 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Tue, 29 Sep 2026 13:19:11 +0200 Subject: [PATCH 13/14] add test for local importer (and rename test class) --- ....st => GitLocalModelImporterTest.class.st} | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) rename src/GitProjectHealth-Model-LocalImporter-Tests/{GitLocalModelImporterWithTestResourceTest.class.st => GitLocalModelImporterTest.class.st} (73%) diff --git a/src/GitProjectHealth-Model-LocalImporter-Tests/GitLocalModelImporterWithTestResourceTest.class.st b/src/GitProjectHealth-Model-LocalImporter-Tests/GitLocalModelImporterTest.class.st similarity index 73% rename from src/GitProjectHealth-Model-LocalImporter-Tests/GitLocalModelImporterWithTestResourceTest.class.st rename to src/GitProjectHealth-Model-LocalImporter-Tests/GitLocalModelImporterTest.class.st index 2f17151a..9ab4d790 100644 --- a/src/GitProjectHealth-Model-LocalImporter-Tests/GitLocalModelImporterWithTestResourceTest.class.st +++ b/src/GitProjectHealth-Model-LocalImporter-Tests/GitLocalModelImporterTest.class.st @@ -2,7 +2,7 @@ This tests uses the test repository under GPHBridge (the files of the repository) " Class { - #name : #GitLocalModelImporterWithTestResourceTest, + #name : #GitLocalModelImporterTest, #superclass : #TestCase, #instVars : [ 'localImporter', @@ -13,13 +13,13 @@ Class { } { #category : #accessing } -GitLocalModelImporterWithTestResourceTest class >> resources [ +GitLocalModelImporterTest class >> resources [ ^ Array with: GitLocalModelImporterWithTestResource ] { #category : #running } -GitLocalModelImporterWithTestResourceTest >> setUp [ +GitLocalModelImporterTest >> setUp [ super setUp. localImporter := GitLocalModelImporter new. @@ -39,7 +39,7 @@ GitLocalModelImporterWithTestResourceTest >> setUp [ ] { #category : #tests } -GitLocalModelImporterWithTestResourceTest >> testConnectionBetweenDivergingBranches [ +GitLocalModelImporterTest >> testConnectionBetweenDivergingBranches [ | lastCommitBranchB | localImporter importRepository: repository. @@ -50,7 +50,7 @@ GitLocalModelImporterWithTestResourceTest >> testConnectionBetweenDivergingBranc ] { #category : #tests } -GitLocalModelImporterWithTestResourceTest >> testConnectionBetweenSuccessiveBranches [ +GitLocalModelImporterTest >> testConnectionBetweenSuccessiveBranches [ | lastCommitBranchB | localImporter importRepository: repository. @@ -66,7 +66,7 @@ GitLocalModelImporterWithTestResourceTest >> testConnectionBetweenSuccessiveBran ] { #category : #tests } -GitLocalModelImporterWithTestResourceTest >> testFirstCommitHasADiff [ +GitLocalModelImporterTest >> testFirstCommitHasADiff [ | commit | localImporter importRepository: repository. @@ -76,7 +76,7 @@ GitLocalModelImporterWithTestResourceTest >> testFirstCommitHasADiff [ ] { #category : #tests } -GitLocalModelImporterWithTestResourceTest >> testHasRenamedField [ +GitLocalModelImporterTest >> testHasRenamedField [ | file renameCommit | localImporter importRepository: repository. @@ -90,7 +90,24 @@ GitLocalModelImporterWithTestResourceTest >> testHasRenamedField [ ] { #category : #tests } -GitLocalModelImporterWithTestResourceTest >> testImportFilesOfBranch [ +GitLocalModelImporterTest >> testImportCommitsOfRepositorySinceUntil [ + + | commits originalSize | + localImporter importBranchesOfRepository: repository. + commits := localImporter + importCommitsOfRepository: repository + since: (DateAndTime year: 2026 month: 9 day: 3) + until: (DateAndTime year: 2026 month: 9 day: 5). + originalSize := commits size. + commits := localImporter + importCommitsOfRepository: repository + since: (DateAndTime year: 2026 month: 8 day: 3) + until: (DateAndTime year: 2026 month: 9 day: 5). + self assert: originalSize < commits size +] + +{ #category : #tests } +GitLocalModelImporterTest >> testImportFilesOfBranch [ | file | localImporter importRepository: repository. @@ -100,7 +117,7 @@ GitLocalModelImporterWithTestResourceTest >> testImportFilesOfBranch [ ] { #category : #tests } -GitLocalModelImporterWithTestResourceTest >> testImportFilesOfBranchDoNotImportNotCommitedFile [ +GitLocalModelImporterTest >> testImportFilesOfBranchDoNotImportNotCommitedFile [ | file | (GPHBridge testResources / 'testResource' / 'wrong-file.st') @@ -113,7 +130,7 @@ GitLocalModelImporterWithTestResourceTest >> testImportFilesOfBranchDoNotImportN ] { #category : #tests } -GitLocalModelImporterWithTestResourceTest >> testSameHeadCommitForTwoBranches [ +GitLocalModelImporterTest >> testSameHeadCommitForTwoBranches [ | branchA anotherBranchA | localImporter importRepository: repository. From 910f678eca2bce12af56999b06f66b9bfa3abd31 Mon Sep 17 00:00:00 2001 From: Benoit Verhaeghe Date: Tue, 29 Sep 2026 13:22:40 +0200 Subject: [PATCH 14/14] fix test --- .../GitLocalModelImporter.class.st | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st index d1661402..af692ada 100644 --- a/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st +++ b/src/GitProjectHealth-Model-LocalImporter/GitLocalModelImporter.class.st @@ -297,7 +297,7 @@ GitLocalModelImporter >> importCommitsOfRepository: aGPHRepository since: sinceD GitLocalModelImporter >> importDiffOfCommit: commit [ | sourceCommit gitDiff | - commit diffs ifNotEmpty: [ + commit diffs ifEmpty: [ sourceCommit := self gitCommitOf: commit. gitDiff := LGitDiff of: sourceCommit repository.