diff --git a/csharp/Platform.Bot/Program.cs b/csharp/Platform.Bot/Program.cs index 521a6b95..dac7acd2 100644 --- a/csharp/Platform.Bot/Program.cs +++ b/csharp/Platform.Bot/Program.cs @@ -98,6 +98,7 @@ private static async Task Main(string[] args) var issueTracker = new IssueTracker(githubStorage, new HelloWorldTrigger(githubStorage, dbContext, fileSetName), new OrganizationLastMonthActivityTrigger(githubStorage), new LastCommitActivityTrigger(githubStorage), new AdminAuthorIssueTriggerDecorator(new ProtectDefaultBranchTrigger(githubStorage), githubStorage), new AdminAuthorIssueTriggerDecorator(new ChangeOrganizationRepositoriesDefaultBranchTrigger(githubStorage, dbContext), githubStorage), new AdminAuthorIssueTriggerDecorator(new ChangeOrganizationPullRequestsBaseBranchTrigger(githubStorage, dbContext), githubStorage)); var pullRequenstTracker = new PullRequestTracker(githubStorage, new MergeDependabotBumpsTrigger(githubStorage)); var timestampTracker = new DateTimeTracker(githubStorage, new CreateAndSaveOrganizationRepositoriesMigrationTrigger(githubStorage, dbContext, Path.Combine(Directory.GetCurrentDirectory(), "/github-migrations"))); + var commitTracker = new CommitTracker(githubStorage, githubUserName, new LoadRepositoryCodeToDoubletsTrigger(githubStorage, dbContext, githubUserName)); var cancellation = new CancellationTokenSource(); while (true) { @@ -105,6 +106,7 @@ private static async Task Main(string[] args) { await issueTracker.Start(cancellation.Token); await pullRequenstTracker.Start(cancellation.Token); + await commitTracker.Start(cancellation.Token); // timestampTracker.Start(cancellation.Token); Thread.Sleep(minimumInteractionInterval); } diff --git a/csharp/Platform.Bot/Trackers/CommitTracker.cs b/csharp/Platform.Bot/Trackers/CommitTracker.cs new file mode 100644 index 00000000..de96e9ec --- /dev/null +++ b/csharp/Platform.Bot/Trackers/CommitTracker.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Interfaces; +using Octokit; +using Storage.Remote.GitHub; + +namespace Platform.Bot.Trackers; + +public class CommitTracker : ITracker +{ + private readonly GitHubStorage _storage; + private readonly string _organizationName; + private readonly IList> _triggers; + + public CommitTracker(GitHubStorage storage, string organizationName, params ITrigger[] triggers) + { + _storage = storage; + _organizationName = organizationName; + _triggers = triggers; + } + + public async Task Start(CancellationToken cancellationToken) + { + var repositories = await _storage.GetAllRepositories(_organizationName); + + foreach (var repository in repositories) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + try + { + // Get recent commits from the default branch + var commitRequest = new CommitRequest + { + Sha = repository.DefaultBranch, + Since = DateTime.Now.AddHours(-1) // Check commits from last hour + }; + + var commits = await _storage.GetCommits(repository.Id, commitRequest); + + foreach (var commit in commits) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + foreach (var trigger in _triggers) + { + if (await trigger.Condition(commit)) + { + await trigger.Action(commit); + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($"Error processing repository {repository.Name}: {ex.Message}"); + } + } + } +} \ No newline at end of file diff --git a/csharp/Platform.Bot/Triggers/LoadRepositoryCodeToDoubletsTrigger.cs b/csharp/Platform.Bot/Triggers/LoadRepositoryCodeToDoubletsTrigger.cs new file mode 100644 index 00000000..b7c00d33 --- /dev/null +++ b/csharp/Platform.Bot/Triggers/LoadRepositoryCodeToDoubletsTrigger.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Interfaces; +using Octokit; +using Storage.Local; +using Storage.Remote.GitHub; + +namespace Platform.Bot.Triggers; + +public class LoadRepositoryCodeToDoubletsTrigger : ITrigger +{ + private readonly GitHubStorage _githubStorage; + private readonly FileStorage _linksStorage; + private readonly string _organizationName; + + public LoadRepositoryCodeToDoubletsTrigger(GitHubStorage githubStorage, FileStorage linksStorage, string organizationName) + { + _githubStorage = githubStorage; + _linksStorage = linksStorage; + _organizationName = organizationName; + } + + public async Task Condition(GitHubCommit commit) + { + // Trigger on every commit to default branch + // We could add more conditions here, like only for specific file extensions, etc. + return true; + } + + public async Task Action(GitHubCommit commit) + { + try + { + var repository = await _githubStorage.Client.Repository.Get(commit.Repository.Id); + + // Create or get file set for this repository + var fileSetName = $"{_organizationName}/{repository.Name}"; + var fileSet = _linksStorage.GetFileSet(fileSetName); + if (fileSet == 0) + { + fileSet = _linksStorage.CreateFileSet(fileSetName); + Console.WriteLine($"Created file set for repository: {fileSetName}"); + } + + // Load all repository content into Doublets store + await LoadRepositoryContentRecursively(repository.Id, "", fileSet, repository.DefaultBranch); + + Console.WriteLine($"Repository code loaded to Doublets store: {fileSetName} (commit: {commit.Sha[..7]})"); + } + catch (Exception ex) + { + Console.WriteLine($"Error loading repository code to Doublets store: {ex.Message}"); + } + } + + private async Task LoadRepositoryContentRecursively(long repositoryId, string path, ulong fileSet, string branch) + { + try + { + var contents = await _githubStorage.GetAllContentsByRef(repositoryId, path, branch); + + foreach (var content in contents) + { + if (content.Type == ContentType.File) + { + // Skip binary files and very large files + if (IsBinaryFile(content.Name) || content.Size > 1024 * 1024) // Skip files > 1MB + { + continue; + } + + try + { + // Get file content (it's base64 encoded) + var fileContent = content.Content; + if (!string.IsNullOrEmpty(fileContent)) + { + // Decode base64 content + var decodedContent = Encoding.UTF8.GetString(Convert.FromBase64String(fileContent)); + + // Store file in Doublets store + var file = _linksStorage.AddFile(decodedContent); + _linksStorage.AddFileToSet(fileSet, file, content.Path); + + Console.WriteLine($"Loaded file: {content.Path}"); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error loading file {content.Path}: {ex.Message}"); + } + } + else if (content.Type == ContentType.Dir) + { + // Recursively load directory contents + await LoadRepositoryContentRecursively(repositoryId, content.Path, fileSet, branch); + } + } + } + catch (Exception ex) + { + Console.WriteLine($"Error loading directory {path}: {ex.Message}"); + } + } + + private bool IsBinaryFile(string fileName) + { + var binaryExtensions = new[] + { + ".exe", ".dll", ".bin", ".pdf", ".jpg", ".jpeg", ".png", ".gif", ".bmp", + ".zip", ".tar", ".gz", ".7z", ".rar", ".ico", ".svg", ".woff", ".woff2", + ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".avi", ".mov", ".wmv", ".wav", + ".ogg", ".flac", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx" + }; + + var extension = System.IO.Path.GetExtension(fileName).ToLower(); + return binaryExtensions.Contains(extension); + } +} \ No newline at end of file diff --git a/csharp/Storage/RemoteStorage/GitHubStorage.cs b/csharp/Storage/RemoteStorage/GitHubStorage.cs index 888a7426..47301837 100644 --- a/csharp/Storage/RemoteStorage/GitHubStorage.cs +++ b/csharp/Storage/RemoteStorage/GitHubStorage.cs @@ -252,6 +252,35 @@ public async Task> GetAuthorIdsOfCommits(long repositoryId, CommitRequ #region Content + public async Task> GetAllContents(long repositoryId, string path = "") + { + return await Client.Repository.Content.GetAllContents(repositoryId, path); + } + + public async Task> GetAllContents(string owner, string repo, string path = "") + { + return await Client.Repository.Content.GetAllContents(owner, repo, path); + } + + public async Task> GetAllContentsByRef(long repositoryId, string path, string reference) + { + return await Client.Repository.Content.GetAllContentsByRef(repositoryId, path, reference); + } + + public async Task GetFileContent(long repositoryId, string path, string reference = null) + { + var contents = reference == null + ? await Client.Repository.Content.GetAllContents(repositoryId, path) + : await Client.Repository.Content.GetAllContentsByRef(repositoryId, path, reference); + + if (contents.Count > 0 && contents[0].Type == ContentType.File) + { + return contents[0].Content; + } + + return null; + } + // public async Task CreateOrUpdateFile(string fileContent, string filePath, Repository repository, string branchName, string commitMessage) // { // try diff --git a/experiments/test_repository_loading.cs b/experiments/test_repository_loading.cs new file mode 100644 index 00000000..e50e4ec4 --- /dev/null +++ b/experiments/test_repository_loading.cs @@ -0,0 +1,55 @@ +// Test script for repository loading functionality +using System; +using System.Threading.Tasks; +using Storage.Local; +using Storage.Remote.GitHub; +using Platform.Bot.Triggers; + +namespace TestRepositoryLoading +{ + class Program + { + static async Task Main(string[] args) + { + // This is a simple test script to verify the repository loading functionality + // Note: This would need actual GitHub credentials to run + + Console.WriteLine("Repository loading test script"); + Console.WriteLine("This tests the new LoadRepositoryCodeToDoubletsTrigger functionality"); + + // Create test database + var testDbPath = "/tmp/test_repository_loading.db"; + var linksStorage = new FileStorage(testDbPath); + + // Test creating file set + var fileSetName = "test/repository"; + var fileSet = linksStorage.CreateFileSet(fileSetName); + Console.WriteLine($"Created file set: {fileSetName} with ID: {fileSet}"); + + // Test adding a file + var testContent = "// This is a test file\nusing System;\n\nnamespace Test\n{\n class Program\n {\n static void Main()\n {\n Console.WriteLine(\"Hello World\");\n }\n }\n}"; + var file = linksStorage.AddFile(testContent); + var fileInSet = linksStorage.AddFileToSet(fileSet, file, "Program.cs"); + Console.WriteLine($"Added test file to set: {fileInSet}"); + + // Test retrieving files from set + var files = linksStorage.GetFilesFromSet(fileSetName); + Console.WriteLine($"Files in set: {files.Count}"); + foreach (var f in files) + { + Console.WriteLine($" Path: {f.Path}"); + Console.WriteLine($" Content length: {f.Content?.Length ?? 0} characters"); + } + + Console.WriteLine("Test completed successfully!"); + + // Clean up + linksStorage.Dispose(); + if (System.IO.File.Exists(testDbPath)) + { + System.IO.File.Delete(testDbPath); + Console.WriteLine("Cleaned up test database"); + } + } + } +} \ No newline at end of file diff --git a/experiments/test_repository_loading.csproj b/experiments/test_repository_loading.csproj new file mode 100644 index 00000000..5ec69efe --- /dev/null +++ b/experiments/test_repository_loading.csproj @@ -0,0 +1,14 @@ + + + + Exe + net8 + enable + + + + + + + + \ No newline at end of file