mugit @ c328fa945ba477f410e888f13738d4c77dda5d57

D

@@ -1,28 +0,0 @@
-server:
-  host: 0.0.0.0
-  port: 8008
-
-meta:
-  description: hey kid, come get your free software
-  title: git.olexsmir.xyz
-  host: git.olexsmir.xyz
-
-ssh:
-  enable: true
-  port: 2222
-  host_key: /home/olex/.ssh/mugit
-  keys:
-    - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPLLJdkVYKZgsayw+sHanKPKZbI0RMS2CakqBCEi5Trz
-    - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMPQ0Qz0DFB+rGrD8ScUqbUTZ1/O8FHrOBF5bIAGQgMj olex@rachlinux
-
-repo:
-  dir: /home/olex/mugit-test/
-  readmes:
-    - README
-    - README.md
-    - readme
-    - readme.md
-    - readme.txt
-  masters:
-    - master
-    - main

M go.mod

@@ -8,6 +8,7 @@ github.com/gliderlabs/ssh v0.3.8
 	github.com/go-git/go-git/v5 v5.16.4
 	github.com/yuin/goldmark v1.7.16
 	golang.org/x/crypto v0.47.0
+	golang.org/x/sync v0.19.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 

M go.sum

@@ -79,6 +79,8 @@ golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
 golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

M internal/config/config.go

@@ -18,17 +18,22 @@ Title       string `yaml:"title"`
 		Description string `yaml:"description"`
 		Host        string `yaml:"host"`
 	} `yaml:"meta"`
+	Repo struct {
+		Dir     string   `yaml:"dir"`
+		Readmes []string `yaml:"readmes"`
+		Masters []string `yaml:"masters"`
+	} `yaml:"repo"`
 	SSH struct {
 		Enable  bool     `yaml:"enable"`
 		Port    int      `yaml:"port"`
 		HostKey string   `yaml:"host_key"`
 		Keys    []string `yaml:"keys"`
 	} `yaml:"ssh"`
-	Repo struct {
-		Dir     string   `yaml:"dir"`
-		Readmes []string `yaml:"readmes"`
-		Masters []string `yaml:"masters"`
-	} `yaml:"repo"`
+	Mirror struct {
+		Enable      bool   `yaml:"enable"`
+		Interval    string `yaml:"interval"`
+		GithubToken string `yaml:"github_token"`
+	} `yaml:"mirror"`
 }
 
 func Load(fpath string) (*Config, error) {

M internal/git/repo.go

@@ -1,14 +1,18 @@
 package git
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
+	"time"
 
 	"github.com/go-git/go-git/v5"
+	gitconfig "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
 )
 
 // Thanks https://git.icyphox.sh/legit/blob/master/git/git.go
@@ -43,6 +47,10 @@ }
 		g.h = *hash
 	}
 	return &g, nil
+}
+
+func (g *Repo) Name() string {
+	return filepath.Base(g.path)
 }
 
 func (g *Repo) Commits() ([]*object.Commit, error) {
@@ -180,3 +188,93 @@ }
 	}
 	return "", fmt.Errorf("unable to find master branch")
 }
+
+type MirrorInfo struct {
+	IsMirror  bool
+	Remote    string
+	RemoteURL string
+}
+
+func (g *Repo) MirrorInfo() (MirrorInfo, error) {
+	c, err := g.r.Config()
+	if err != nil {
+		return MirrorInfo{}, fmt.Errorf("failed to read config: %w", err)
+	}
+
+	isMirror := c.Raw.Section("mugit").Options.Get("mirror") == "true"
+	for _, remote := range c.Remotes {
+		if len(remote.URLs) > 0 && (remote.Name == "upstream" || remote.Name == "origin") {
+			return MirrorInfo{
+				IsMirror:  isMirror,
+				Remote:    remote.Name,
+				RemoteURL: remote.URLs[0],
+			}, nil
+		}
+	}
+	// TODO: error if mirror opt is set, but there's no remotes
+	return MirrorInfo{}, fmt.Errorf("no mirror remote found")
+}
+
+func (g *Repo) ReadLastSync() (time.Time, error) {
+	c, err := g.r.Config()
+	if err != nil {
+		return time.Time{}, fmt.Errorf("failed to read config: %w", err)
+	}
+
+	raw := c.Raw.Section("mugit").Options.Get("last-sync")
+	if raw == "" {
+		return time.Time{}, fmt.Errorf("last-sync not set")
+	}
+
+	out, err := time.Parse(time.RFC3339, string(raw))
+	if err != nil {
+		return time.Time{}, fmt.Errorf("failed to parse time: %w", err)
+	}
+	return out, nil
+}
+
+func (g *Repo) SetLastSync(lastSync time.Time) error {
+	c, err := g.r.Config()
+	if err != nil {
+		return fmt.Errorf("failed to read config: %w", err)
+	}
+
+	c.Raw.Section("mugit").
+		SetOption("last-sync", lastSync.Format(time.RFC3339))
+	return g.r.SetConfig(c)
+}
+
+func (g *Repo) Fetch(remote string) error {
+	return g.FetchWithAuth(remote, "")
+}
+
+// FetchWithAuth fetches but with auth. Works only with github's auth
+func (g *Repo) FetchWithAuth(remote string, token string) error {
+	rmt, err := g.r.Remote(remote)
+	if err != nil {
+		return fmt.Errorf("failed to get upstream remote: %w", err)
+	}
+
+	opts := &git.FetchOptions{
+		RefSpecs: []gitconfig.RefSpec{
+			// fetch all branches
+			"+refs/heads/*:refs/heads/*",
+			"+refs/tags/*:refs/tags/*",
+		},
+		Tags:  git.AllTags,
+		Prune: true,
+		Force: true,
+	}
+
+	if token != "" {
+		opts.Auth = &http.BasicAuth{
+			Username: token,
+			Password: "x-oauth-basic",
+		}
+	}
+
+	if err := rmt.Fetch(opts); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
+		return fmt.Errorf("fetch failed: %w", err)
+	}
+	return nil
+}

M internal/handlers/repo.go

@@ -34,6 +34,10 @@ }
 
 	repoInfos := []repoInfo{}
 	for _, dir := range dirs {
+		if !dir.IsDir() {
+			continue
+		}
+
 		name := dir.Name()
 		repo, err := h.openPublicRepo(name, "")
 		if err != nil {

A internal/mirror/mirror.go

@@ -0,0 +1,165 @@
+package mirror
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/sync/semaphore"
+	"olexsmir.xyz/mugit/internal/config"
+	"olexsmir.xyz/mugit/internal/git"
+)
+
+type Worker struct {
+	c *config.Config
+}
+
+func NewWorker(cfg *config.Config) *Worker {
+	return &Worker{
+		c: cfg,
+	}
+}
+
+func (w *Worker) Start(ctx context.Context) error {
+	interval, err := time.ParseDuration(w.c.Mirror.Interval)
+	if err != nil {
+		slog.Error("couldn't parse interval time", "err", err)
+		return err
+	}
+
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	if err := w.mirror(ctx); err != nil {
+		slog.Error("initial mirror sync failed", "err", err)
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+			if err := w.mirror(ctx); err != nil {
+				slog.Error("mirror sync failed", "err", err)
+			}
+		}
+	}
+}
+
+func (w *Worker) mirror(ctx context.Context) error {
+	repos, err := w.findMirrorRepos()
+	if err != nil {
+		return err
+	}
+
+	var wg sync.WaitGroup
+	sem := semaphore.NewWeighted(5)
+	errCh := make(chan error, len(repos))
+
+	for _, repo := range repos {
+		wg.Go(func() {
+			if err := sem.Acquire(ctx, 1); err != nil {
+				errCh <- err
+				return
+			}
+			defer sem.Release(1)
+
+			if err := w.syncRepo(ctx, repo); err != nil {
+				errCh <- err
+			}
+		})
+	}
+	wg.Wait()
+	close(errCh)
+
+	var errs []error
+	for err := range errCh {
+		errs = append(errs, err)
+	}
+	return errors.Join(errs...)
+}
+
+func (w *Worker) syncRepo(_ context.Context, repo *git.Repo) error {
+	name := repo.Name()
+	slog.Info("mirror: sync started", "repo", name)
+
+	mi, err := repo.MirrorInfo()
+	if err != nil {
+		slog.Error("mirror: failed to get info", "repo", name, "err", err)
+		return err
+	}
+
+	if err := w.isRemoteValid(mi.RemoteURL); err != nil {
+		slog.Error("mirror: remote is not valid", "repo", name, "err", err)
+		return err
+	}
+
+	if w.isRemoteGithub(mi.RemoteURL) && w.c.Mirror.GithubToken != "" {
+		if err := repo.FetchWithAuth(mi.Remote, w.c.Mirror.GithubToken); err != nil {
+			slog.Error("mirror: fetch failed (authorized)", "repo", name, "err", err)
+			return err
+		}
+	} else {
+		if err := repo.Fetch(mi.Remote); err != nil {
+			slog.Error("mirror: fetch failed", "repo", name, "err", err)
+			return err
+		}
+	}
+
+	if err := repo.SetLastSync(time.Now()); err != nil {
+		slog.Error("mirror: failed to set last sync time", "repo", name, "err", err)
+	}
+
+	slog.Info("mirror: sync completed", "repo", repo.Name())
+	return nil
+}
+
+func (w *Worker) findMirrorRepos() ([]*git.Repo, error) {
+	dirs, err := os.ReadDir(w.c.Repo.Dir)
+	if err != nil {
+		return nil, err
+	}
+
+	var repos []*git.Repo
+	for _, dir := range dirs {
+		if !dir.IsDir() {
+			continue
+		}
+
+		name := dir.Name()
+		repo, err := git.Open(filepath.Join(w.c.Repo.Dir, filepath.Clean(name)), "")
+		if err != nil {
+			slog.Debug("skipping non-git directory", "path", name, "err", err)
+			continue
+		}
+
+		mirror, err := repo.MirrorInfo()
+		if err != nil {
+			slog.Debug("skipping non-mirror repo", "path", name, "err", err)
+			continue
+		}
+
+		if mirror.IsMirror {
+			repos = append(repos, repo)
+		}
+	}
+
+	return repos, nil
+}
+
+func (w *Worker) isRemoteValid(remote string) error {
+	if !strings.HasPrefix(remote, "http") {
+		return fmt.Errorf("only http and https remotes are supported")
+	}
+	return nil
+}
+
+func (w *Worker) isRemoteGithub(remoteURL string) bool {
+	return strings.Contains(remoteURL, "github.com")
+}

M main.go

@@ -13,6 +13,7 @@ "syscall"
 
 	"olexsmir.xyz/mugit/internal/config"
 	"olexsmir.xyz/mugit/internal/handlers"
+	"olexsmir.xyz/mugit/internal/mirror"
 	"olexsmir.xyz/mugit/internal/ssh"
 )
 
@@ -24,7 +25,7 @@ }
 }
 
 func run() error {
-	cfg, err := config.Load("/home/olex/code/mugit/config.yml")
+	cfg, err := config.Load("/home/olex/mugit-test/config.yml")
 	if err != nil {
 		slog.Error("config error", "err", err)
 		return err
@@ -45,10 +46,20 @@ }()
 
 	sshServer := ssh.NewServer(cfg)
 	if cfg.SSH.Enable {
-		slog.Info("starting ssh server", "port", cfg.SSH.Port)
-		if err := sshServer.Start(); err != nil {
-			slog.Error("ssh server error", "err", err)
-		}
+		go func() {
+			slog.Info("starting ssh server", "port", cfg.SSH.Port)
+			if err := sshServer.Start(); err != nil {
+				slog.Error("ssh server error", "err", err)
+			}
+		}()
+	}
+
+	mirrorer := mirror.NewWorker(cfg)
+	if cfg.Mirror.Enable {
+		go func() {
+			slog.Info("starting mirroring worker")
+			mirrorer.Start(context.TODO())
+		}()
 	}
 
 	// Wait for interrupt signal