diff --git a/__tests__/history.integration.test.ts b/__tests__/history.integration.test.ts index f9b2228..9fd30aa 100644 --- a/__tests__/history.integration.test.ts +++ b/__tests__/history.integration.test.ts @@ -1,5 +1,5 @@ import {analyseHistory} from '../src/commands/history/history' -jest.setTimeout(30000); +jest.setTimeout(100000000); describe('test history analysis for default plugins', () => { it('test history analysis for javascript and ruby', async () => { diff --git a/src/commands/history/history.ts b/src/commands/history/history.ts index 67570a6..f6d8038 100644 --- a/src/commands/history/history.ts +++ b/src/commands/history/history.ts @@ -1,12 +1,14 @@ import {Command} from 'commander' import git from 'isomorphic-git' -import fs from 'fs' import {AnalyseOptions} from '../analyse' import {getPluginsFromNames} from '../../plugins' import {DepinderProject} from "../../extension-points/extract" import {Plugin} from "../../extension-points/plugin" import minimatch from "minimatch"; import {log} from "../../utils/logging" +import path from "path" +import {depinderTempFolder} from "../../utils/utils" +import {promises as fs} from 'fs' export const historyCommand = new Command() .name('history') @@ -36,31 +38,68 @@ export async function analyseHistory(folders: string[], options: AnalyseOptions, } } +// Function to fetch commits from a Git repository +async function getCommits(folder: string): Promise { + try { + return await git.log({ + fs, + dir: folder, + }); + } catch (error) { + console.error(`Failed to get commits for ${folder}:`, error); + return []; + } +} + // Function to process each commit and update the map by plugin -async function processCommitForPlugins(commit: any, folder: string, selectedPlugins: Plugin[], projectMap: Map) { - console.log('Commit:', JSON.stringify(commit, null, 2)); +async function processCommitForPlugins(commit: any, folder: string, selectedPlugins: Plugin[], projectMap: Map +) { const changes = await getChangedFiles(commit, folder); - console.log("Changes: ", changes); + await ensureDirectoryExists(depinderTempFolder); for (const plugin of selectedPlugins) { - const filteredFiles = changes - .filter(file => plugin.extractor.filter ? plugin.extractor.filter(file) : true) - .filter(file => plugin.extractor.files.some(pattern => minimatch(file, pattern, {matchBase: true}))); - console.log("Filtered Files: " + filteredFiles); + let filteredFiles = changes + .filter((file) => (plugin.extractor.filter ? plugin.extractor.filter(file) : true)) + .filter((file) => plugin.extractor.files.some((pattern) => minimatch(file, pattern, { matchBase: true }))); + + if (filteredFiles.includes('package.json') || filteredFiles.includes('package-lock.json') || filteredFiles.includes('manifest.json')) { + if (!filteredFiles.includes('package.json')) { + filteredFiles.push('package.json'); + } + if (!filteredFiles.includes('package-lock.json')) { + filteredFiles.push('package-lock.json'); + } + } + + console.log('Filtered Files: ' + filteredFiles); if (filteredFiles.length > 0) { - // Create DepinderProjects - const projects: DepinderProject[] = await extractProjects(plugin, filteredFiles, commit, folder); - - for (const project of projects) { - console.log("projects: " + projects.length) - console.log(`Plugin ${plugin.name} analyzing project ${project.name}@${project.version}`) - if (project.dependencies) { - console.log(`Dependencies for project in commit ${commit.oid}:`, project.dependencies); - } else { - console.log(`No dependencies found for project in commit ${commit.oid}`); + const tempFilePaths: string[] = []; + for (const file of filteredFiles) { + const tempFilePath = path.join(depinderTempFolder, `${commit.oid}-${path.basename(file)}`); + try { + const fileContent = await git.readBlob({ + fs, + dir: folder, + oid: commit.oid, + filepath: file, + }); + + await fs.writeFile(tempFilePath, fileContent.blob); + tempFilePaths.push(tempFilePath); + } catch (error) { + console.error(`Failed to create temp file for ${file} at commit ${commit.oid}:`, error); } } + + const projects: DepinderProject[] = await extractProjects(plugin, tempFilePaths); + + projects.forEach(project => { + const dependencyIds = Object.keys(project.dependencies); + console.log(`Project: ${project.name}, Dependencies: ${dependencyIds}`); + }); + + await cleanupTempFiles(tempFilePaths); } } } @@ -101,82 +140,44 @@ async function getChangedFiles(commit: any, folder: string): Promise { return changedFiles; } -// Function to fetch commits from a Git repository -async function getCommits(folder: string): Promise { - try { - return await git.log({ - fs, - dir: folder, - }); - } catch (error) { - console.error(`Failed to get commits for ${folder}:`, error); - return []; - } -} - -async function extractProjects(plugin: Plugin, files: string[], commit: any, folder: string): Promise { - const projects: DepinderProject[] = []; +async function extractProjects(plugin: Plugin, files: string[]) { + const projects = [] as DepinderProject[] for (const context of plugin.extractor.createContexts(files)) { - log.info(`Parsing dependency tree information for context: ${JSON.stringify(context)}`); - - const manifestFile = context.manifestFile || ''; - let manifestContent: any; - log.info(manifestFile); - - if (manifestFile) { - try { - // Read the manifest file from the specific commit - const manifestBuffer = await git.readBlob({ - fs, - dir: folder, - oid: commit.oid, - filepath: manifestFile - }); - - // Convert the Uint8Array buffer to a UTF-8 string - const rawContent = new TextDecoder("utf-8").decode(manifestBuffer.blob); - - // Log the raw content for debugging purposes - log.info(`Raw content of ${manifestFile} from commit ${commit.oid}:\n${rawContent}`); - - // Try to parse the content as JSON directly - manifestContent = JSON.parse(rawContent); - } catch (error) { - // If JSON parsing fails or if the file cannot be read, log the error - if (error instanceof SyntaxError) { - log.error(`Manifest file ${manifestFile} in commit ${commit.oid} is not valid JSON: ${error.message}`); - } else { - log.error(`Failed to read or parse ${manifestFile} from commit ${commit.oid}:`, error); - } - continue; // Skip this context if we can't read or parse the manifest file + log.info(`Parsing dependency tree information for ${JSON.stringify(context)}`) + try { + if (!plugin.parser) { + log.info(`Plugin ${plugin.name} does not have a parser!`) + continue } - } else { - log.warn(`No manifest file defined in context for plugin ${plugin.name}`); - continue; + const proj: DepinderProject = await plugin.parser.parseDependencyTree(context) + log.info(`Done parsing dependency tree information for ${JSON.stringify(context)}`) + projects.push(proj) + } catch (e: any) { + log.warn(`Exception parsing dependency tree information for ${JSON.stringify(context)}`) + log.error(e) } + } + return projects +} - // Now that we have the manifest content, extract the project name and version - const projectName = manifestContent.name || `Unnamed Project (commit: ${commit.oid})`; - const projectVersion = manifestContent.version || '0.0.0'; - try { - if (!plugin.parser) { - log.warn(`Plugin ${plugin.name} does not have a parser, skipping...`); - continue; - } +// Function to ensure the directory exists +async function ensureDirectoryExists(directoryPath: string) { + try { + await fs.mkdir(directoryPath, { recursive: true }); + } catch (error) { + console.error(`Failed to create directory: ${directoryPath}`, error); + } +} - // Parse dependencies from the context - const proj: DepinderProject = await plugin.parser.parseDependencyTree(context); - proj.name = projectName; - proj.version = projectVersion; - log.info(`Successfully parsed dependency tree for ${projectName}@${projectVersion}`); - projects.push(proj); - } catch (error: any) { - log.error(`Error while parsing dependency tree for project in commit ${commit.oid}:`, error); +// Function to clean up temp files after processing +async function cleanupTempFiles(tempFilePaths: string[]) { + for (const filePath of tempFilePaths) { + try { + await fs.unlink(filePath); + } catch (error) { } } - - return projects; } diff --git a/src/plugins/javascript/index.ts b/src/plugins/javascript/index.ts index 51afed3..b166e8f 100644 --- a/src/plugins/javascript/index.ts +++ b/src/plugins/javascript/index.ts @@ -23,7 +23,7 @@ const extractor: Extractor = { const lockFileContexts = files.filter(it => it.endsWith('package-lock.json') || it.endsWith('yarn.lock')).map(it => ({ root: path.dirname(it), lockFile: path.basename(it), - manifestFile: 'package.json', + manifestFile: files.find(manifest => path.dirname(manifest) === path.dirname(it) && manifest.endsWith('package.json')), } as DependencyFileContext)) const packageJsonWithLockInParent = files.filter(it => it.endsWith('package.json')) @@ -116,7 +116,7 @@ function transformDeps(tree: DepTreeDep, root: string): Map { // const lockFileVersion = getLockfileVersionFromFile(lockFile) // log.info(`parsing ${path.resolve(root, lockFile)}`) - const result = await buildDepTreeFromFiles(root, manifestFile ?? 'package.json', lockFile ?? '', true); + const result = await buildDepTreeFromFiles(root, manifestFile ?? 'package.json', lockFile ?? '', true, false); const manifestJSON = JSON.parse(fs.readFileSync(path.resolve(root, manifestFile ?? 'package.json'), 'utf8')) return {