Allow GitHub bot to load repository code to Doublets store on default branch commits

csharp/Platform.Bot/Program.cs

@@ -98,13 +98,15 @@ private static async Task<int> Main(string[] args)
                 var issueTracker = new IssueTracker(githubStorage, new HelloWorldTrigger(githubStorage, dbContext, fileSetName), new OrganizationLastMonthActivityTrigger(githubStorage), new LastCommitActivityTrigger(githubStorage), new AdminAuthorIssueTriggerDecorator(new ProtectDefaultBranchTrigger(githubStorage), githubStorage), new AdminAuthorIssueTriggerDecorator(new ChangeOrganizationRepositoriesDefaultBranchTrigger(githubStorage, dbContext), githubStorage), new AdminAuthorIssueTriggerDecorator(new ChangeOrganizationPullRequestsBaseBranchTrigger(githubStorage, dbContext), githubStorage));
                 var pullRequenstTracker = new PullRequestTracker(githubStorage, new MergeDependabotBumpsTrigger(githubStorage));
                 var timestampTracker = new DateTimeTracker(githubStorage, new CreateAndSaveOrganizationRepositoriesMigrationTrigger(githubStorage, dbContext, Path.Combine(Directory.GetCurrentDirectory(), "/github-migrations")));
+                var commitTracker = new CommitTracker(githubStorage, githubUserName, new LoadRepositoryCodeToDoubletsTrigger(githubStorage, dbContext, githubUserName));
                 var cancellation = new CancellationTokenSource();
                 while (true)
                 {
                     try
                     {
                         await issueTracker.Start(cancellation.Token);
                         await pullRequenstTracker.Start(cancellation.Token);
+                        await commitTracker.Start(cancellation.Token);
                         // timestampTracker.Start(cancellation.Token);
                         Thread.Sleep(minimumInteractionInterval);
                     }

csharp/Platform.Bot/Trackers/CommitTracker.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Interfaces;
+using Octokit;
+using Storage.Remote.GitHub;
+
+namespace Platform.Bot.Trackers;
+
+public class CommitTracker : ITracker<GitHubCommit>
+{
+    private readonly GitHubStorage _storage;
+    private readonly string _organizationName;
+    private readonly IList<ITrigger<GitHubCommit>> _triggers;
+
+    public CommitTracker(GitHubStorage storage, string organizationName, params ITrigger<GitHubCommit>[] triggers)
+    {
+        _storage = storage;
+        _organizationName = organizationName;
+        _triggers = triggers;
+    }
+
+    public async Task Start(CancellationToken cancellationToken)
+    {
+        var repositories = await _storage.GetAllRepositories(_organizationName);
+
+        foreach (var repository in repositories)
+        {
+            if (cancellationToken.IsCancellationRequested)
+            {
+                return;
+            }
+
+            try
+            {
+                // Get recent commits from the default branch
+                var commitRequest = new CommitRequest 
+                { 
+                    Sha = repository.DefaultBranch,
+                    Since = DateTime.Now.AddHours(-1) // Check commits from last hour
+                };
+
+                var commits = await _storage.GetCommits(repository.Id, commitRequest);
+
+                foreach (var commit in commits)
+                {
+                    if (cancellationToken.IsCancellationRequested)
+                    {
+                        return;
+                    }
+
+                    foreach (var trigger in _triggers)
+                    {
+                        if (await trigger.Condition(commit))
+                        {
+                            await trigger.Action(commit);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Error processing repository {repository.Name}: {ex.Message}");
+            }
+        }
+    }
+}

csharp/Platform.Bot/Triggers/LoadRepositoryCodeToDoubletsTrigger.cs

@@ -0,0 +1,122 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Interfaces;
+using Octokit;
+using Storage.Local;
+using Storage.Remote.GitHub;
+
+namespace Platform.Bot.Triggers;
+
+public class LoadRepositoryCodeToDoubletsTrigger : ITrigger<GitHubCommit>
+{
+    private readonly GitHubStorage _githubStorage;
+    private readonly FileStorage _linksStorage;
+    private readonly string _organizationName;
+
+    public LoadRepositoryCodeToDoubletsTrigger(GitHubStorage githubStorage, FileStorage linksStorage, string organizationName)
+    {
+        _githubStorage = githubStorage;
+        _linksStorage = linksStorage;
+        _organizationName = organizationName;
+    }
+
+    public async Task<bool> Condition(GitHubCommit commit)
+    {
+        // Trigger on every commit to default branch
+        // We could add more conditions here, like only for specific file extensions, etc.
+        return true;
+    }
+
+    public async Task Action(GitHubCommit commit)
+    {
+        try
+        {
+            var repository = await _githubStorage.Client.Repository.Get(commit.Repository.Id);
+
+            // Create or get file set for this repository
+            var fileSetName = $"{_organizationName}/{repository.Name}";
+            var fileSet = _linksStorage.GetFileSet(fileSetName);
+            if (fileSet == 0)
+            {
+                fileSet = _linksStorage.CreateFileSet(fileSetName);
+                Console.WriteLine($"Created file set for repository: {fileSetName}");
+            }
+
+            // Load all repository content into Doublets store
+            await LoadRepositoryContentRecursively(repository.Id, "", fileSet, repository.DefaultBranch);
+
+            Console.WriteLine($"Repository code loaded to Doublets store: {fileSetName} (commit: {commit.Sha[..7]})");
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error loading repository code to Doublets store: {ex.Message}");
+        }
+    }
+
+    private async Task LoadRepositoryContentRecursively(long repositoryId, string path, ulong fileSet, string branch)
+    {
+        try
+        {
+            var contents = await _githubStorage.GetAllContentsByRef(repositoryId, path, branch);
+
+            foreach (var content in contents)
+            {
+                if (content.Type == ContentType.File)
+                {
+                    // Skip binary files and very large files
+                    if (IsBinaryFile(content.Name) || content.Size > 1024 * 1024) // Skip files > 1MB
+                    {
+                        continue;
+                    }
+
+                    try
+                    {
+                        // Get file content (it's base64 encoded)
+                        var fileContent = content.Content;
+                        if (!string.IsNullOrEmpty(fileContent))
+                        {
+                            // Decode base64 content
+                            var decodedContent = Encoding.UTF8.GetString(Convert.FromBase64String(fileContent));
+
+                            // Store file in Doublets store
+                            var file = _linksStorage.AddFile(decodedContent);
+                            _linksStorage.AddFileToSet(fileSet, file, content.Path);
+
+                            Console.WriteLine($"Loaded file: {content.Path}");
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        Console.WriteLine($"Error loading file {content.Path}: {ex.Message}");
+                    }
+                }
+                else if (content.Type == ContentType.Dir)
+                {
+                    // Recursively load directory contents
+                    await LoadRepositoryContentRecursively(repositoryId, content.Path, fileSet, branch);
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error loading directory {path}: {ex.Message}");
+        }
+    }
+
+    private bool IsBinaryFile(string fileName)
+    {
+        var binaryExtensions = new[]
+        {
+            ".exe", ".dll", ".bin", ".pdf", ".jpg", ".jpeg", ".png", ".gif", ".bmp", 
+            ".zip", ".tar", ".gz", ".7z", ".rar", ".ico", ".svg", ".woff", ".woff2",
+            ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".avi", ".mov", ".wmv", ".wav",
+            ".ogg", ".flac", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx"
+        };
+
+        var extension = System.IO.Path.GetExtension(fileName).ToLower();
+        return binaryExtensions.Contains(extension);
+    }
+}

csharp/Storage/RemoteStorage/GitHubStorage.cs

@@ -252,6 +252,35 @@ public async Task<List<int>> GetAuthorIdsOfCommits(long repositoryId, CommitRequ
 
         #region Content
 
+        public async Task<IReadOnlyList<RepositoryContent>> GetAllContents(long repositoryId, string path = "")
+        {
+            return await Client.Repository.Content.GetAllContents(repositoryId, path);
+        }
+
+        public async Task<IReadOnlyList<RepositoryContent>> GetAllContents(string owner, string repo, string path = "")
+        {
+            return await Client.Repository.Content.GetAllContents(owner, repo, path);
+        }
+
+        public async Task<IReadOnlyList<RepositoryContent>> GetAllContentsByRef(long repositoryId, string path, string reference)
+        {
+            return await Client.Repository.Content.GetAllContentsByRef(repositoryId, path, reference);
+        }
+
+        public async Task<string> GetFileContent(long repositoryId, string path, string reference = null)
+        {
+            var contents = reference == null 
+                ? await Client.Repository.Content.GetAllContents(repositoryId, path)
+                : await Client.Repository.Content.GetAllContentsByRef(repositoryId, path, reference);
+
+            if (contents.Count > 0 && contents[0].Type == ContentType.File)
+            {
+                return contents[0].Content;
+            }
+
+            return null;
+        }
+
         // public async Task<RepositoryContentChangeSet> CreateOrUpdateFile(string fileContent, string filePath, Repository repository, string branchName, string commitMessage)
         // {
         //     try

experiments/test_repository_loading.cs

@@ -0,0 +1,55 @@
+// Test script for repository loading functionality
+using System;
+using System.Threading.Tasks;
+using Storage.Local;
+using Storage.Remote.GitHub;
+using Platform.Bot.Triggers;
+
+namespace TestRepositoryLoading
+{
+    class Program
+    {
+        static async Task Main(string[] args)
+        {
+            // This is a simple test script to verify the repository loading functionality
+            // Note: This would need actual GitHub credentials to run
+
+            Console.WriteLine("Repository loading test script");
+            Console.WriteLine("This tests the new LoadRepositoryCodeToDoubletsTrigger functionality");
+
+            // Create test database
+            var testDbPath = "/tmp/test_repository_loading.db";
+            var linksStorage = new FileStorage(testDbPath);
+
+            // Test creating file set
+            var fileSetName = "test/repository";
+            var fileSet = linksStorage.CreateFileSet(fileSetName);
+            Console.WriteLine($"Created file set: {fileSetName} with ID: {fileSet}");
+
+            // Test adding a file
+            var testContent = "// This is a test file\nusing System;\n\nnamespace Test\n{\n    class Program\n    {\n        static void Main()\n        {\n            Console.WriteLine(\"Hello World\");\n        }\n    }\n}";
+            var file = linksStorage.AddFile(testContent);
+            var fileInSet = linksStorage.AddFileToSet(fileSet, file, "Program.cs");
+            Console.WriteLine($"Added test file to set: {fileInSet}");
+
+            // Test retrieving files from set
+            var files = linksStorage.GetFilesFromSet(fileSetName);
+            Console.WriteLine($"Files in set: {files.Count}");
+            foreach (var f in files)
+            {
+                Console.WriteLine($"  Path: {f.Path}");
+                Console.WriteLine($"  Content length: {f.Content?.Length ?? 0} characters");
+            }
+
+            Console.WriteLine("Test completed successfully!");
+
+            // Clean up
+            linksStorage.Dispose();
+            if (System.IO.File.Exists(testDbPath))
+            {
+                System.IO.File.Delete(testDbPath);
+                Console.WriteLine("Cleaned up test database");
+            }
+        }
+    }
+}

experiments/test_repository_loading.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../csharp/Platform.Bot/Platform.Bot.csproj" />
+    <ProjectReference Include="../csharp/Storage/Storage.csproj" />
+  </ItemGroup>
+
+</Project>