#1 scan all projects for last modified

go.mod

@@ -1,9 +1,9 @@
 module git.entr0py.de/garionion/murphy
 
-go 1.16
+go 1.17
 
 require (
-	github.com/karrick/godirwalk v1.16.1 // indirect
+	github.com/karrick/godirwalk v1.16.1
 	github.com/mehdioa/nlog v0.0.0-20210327090009-d60bf476a16a
-	github.com/oriser/regroup v0.0.0-20210730155327-fca8d7531263 // indirect
+	github.com/oriser/regroup v0.0.0-20210730155327-fca8d7531263
 )

main.go

@@ -4,13 +4,13 @@ import (
 	"git.entr0py.de/garionion/murphy/scanner"
 	"git.entr0py.de/garionion/murphy/store"
 	"github.com/mehdioa/nlog"
+	"sync"
 )
 
 var log *nlog.Logger
 
 const source = "/mnt/clarkson"
 
-
 func main() {
 	formatter := nlog.NewTextFormatter(true, true)
 	log = nlog.NewLogger(nlog.DebugLevel, formatter)
@@ -18,4 +18,8 @@ func main() {
 	s := store.NewStore(log)
 
 	scanner.InitScanner(s, source)
+
+	wg := new(sync.WaitGroup)
+	wg.Add(2)
+	wg.Wait()
 }

scanner/scanner.go

@@ -1,56 +1,57 @@
 package scanner
 
 import (
-	"fmt"
+	"git.entr0py.de/garionion/murphy/store"
-	store "git.entr0py.de/garionion/murphy/store"
 	"github.com/karrick/godirwalk"
 	"github.com/mehdioa/nlog"
 	"github.com/oriser/regroup"
 	"io/ioutil"
+	"os"
 	"path"
-	"strings"
 	"time"
 )
 
-type Scanner struct {
+func scan(p *store.Project) {
-	TargetPath string
+	log := p.Logger
-	LastChange time.Time
+	log.Infof("Scanning %s", p.ProjectName)
-	Created    time.Time
+	lastModified := time.Time{}
-	ProjectName string
+	err := godirwalk.Walk(p.TargetPath, &godirwalk.Options{
-	Logger     *nlog.Logger
-}
-
-func NewScanner(target, projectName string, createdAt time.Time, logger *nlog.Logger) *Scanner {
-	projectLogger := logger.New("Scanner", nlog.Data{"Project": projectName,})
-	scanner := &Scanner{TargetPath: target, Created: createdAt, ProjectName: projectName, Logger: logger}
-	projectLogger.Infof("Scanning %s", projectName)
-	err := godirwalk.Walk(target, &godirwalk.Options{
 		Callback: func(osPathname string, de *godirwalk.Dirent) error {
-			// Following string operation is not most performant way
+			fi, err := os.Stat(osPathname)
-			// of doing this, but common enough to warrant a simple
+			if err != nil {
-			// example here:
+				log.Errorf("Error while getting Stats on %s: %v", osPathname, err)
-			if strings.Contains(osPathname, ".git") {
+				return err
-				return godirwalk.SkipThis
+			}
+			modtime := fi.ModTime()
+			if modtime.After(lastModified) {
+				lastModified = modtime
 			}
-			fmt.Printf("%s %s\n", de.ModeType(), osPathname)
 			return nil
 		},
 		Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc)
 	})
 	if err != nil {
-		projectLogger.Errorf("Error scanning %s: %s", projectName, err)
+		log.Errorf("Error scanning %s: %v", p.ProjectName, err)
 	}
-	return scanner
+	log.Infof("Found last modification time %v", lastModified)
+	p.LastChange = lastModified
 }
 
+func NewProject(target, projectName string, createdAt time.Time, logger *nlog.Logger) *store.Project {
+	projectLogger := logger.New("Project", nlog.Data{"Project": projectName})
+	project := &store.Project{TargetPath: target, Created: createdAt, ProjectName: projectName, Logger: projectLogger}
+	go scan(project)
+	return project
+}
 
 func InitScanner(s *store.Store, sourcePath string) {
-    s.Logger.Infof("Initializing scanner")
+	s.Logger.Infof("Initializing scanner")
-    scanners := map[string]*Scanner{}
+	projects := s.GetProjects()
-    RootScanner(sourcePath, scanners, s.Logger)
+	rootScanner(sourcePath, projects, s.Logger)
+	s.Logger.Infof("Found %d projects", len(projects))
 }
 
-func RootScanner(rootPath string, scanners map[string]*Scanner, logger *nlog.Logger) {
+func rootScanner(rootPath string, projects map[string]*store.Project, logger *nlog.Logger) {
 	resorts, err := ioutil.ReadDir(rootPath)
 	if err != nil {
 		logger.Errorf("Error reading directory %s: %v", rootPath, err)
@@ -61,14 +62,14 @@ func RootScanner(rootPath string, scanners map[string]*Scanner, logger *nlog.Log
 		if r.IsDir() && r.Name()[0] != '.' {
 			resort := path.Join(rootPath, r.Name())
 			logger.Debugf("Found directory: %s", resort)
-			projects, err := ioutil.ReadDir(resort)
+			projectsDir, err := ioutil.ReadDir(resort)
 			if err != nil {
 				logger.Errorf("Error reading directory %s: %v", projects, err)
 				return
 			}
-			for _, p := range projects {
+			for _, p := range projectsDir {
 				if p.IsDir() {
-					_, mapContainsProject := scanners[p.Name()]
+					_, mapContainsProject := projects[p.Name()]
 					if mapContainsProject {
 						continue
 					}
@@ -86,15 +87,17 @@ func RootScanner(rootPath string, scanners map[string]*Scanner, logger *nlog.Log
 
 						createdAt, err := time.Parse("2006-01-02", createdAtString)
 						if err != nil {
-							logger.Errorf("Failed to parse date %s: %v, skipping %s", createdAtString, err, p.Name())
+							logger.Errorf("Failed to parse date %s: %v, Using Null-Time for %s", createdAtString, err, p.Name())
+							createdAt = time.Time{}
 						}
-						scanner := NewScanner(projectPath, projectName, createdAt, logger)
+						scanner := NewProject(projectPath, projectName, createdAt, logger)
-						scanners[p.Name()] = scanner
+						projects[p.Name()] = scanner
 					} else {
 						logger.Debugf("Ignoring Directory %s", projectPath)
 					}
-                }
+				}
 			}
 		}
 	}
+	return
 }

store/store.go

@@ -2,18 +2,33 @@ package store
 
 import (
 	"github.com/mehdioa/nlog"
+	"sync"
 	"time"
 )
 
 type Store struct {
-	TargetPath []string
+	TargetPath  []string
-	SourcePath []string
+	SourcePath  []string
 	GracePeriod time.Duration
-	Logger	*nlog.Logger
+	Projects    map[string]*Project
+	Logger      *nlog.Logger
+	lock        sync.RWMutex
 }
 
-func NewStore(Logger *nlog.Logger)  *Store{
+type Project struct {
-	store := &Store{Logger: Logger}
+	TargetPath  string
+	LastChange  time.Time
+	Created     time.Time
+	ProjectName string
+	Logger      *nlog.Node
+}
+
+func NewStore(Logger *nlog.Logger) *Store {
+	store := &Store{Logger: Logger, Projects: make(map[string]*Project)}
 
 	return store
 }
+
+func (store *Store) GetProjects() map[string]*Project {
+	return store.Projects
+}