murphy/scanner/scanner.go

package scanner

import (
	"git.entr0py.de/garionion/murphy/store"
	"github.com/go-co-op/gocron"
	"github.com/hako/durafmt"
	"github.com/karrick/godirwalk"
	"github.com/mehdioa/nlog"
	"github.com/oriser/regroup"
	"io/ioutil"
	"os"
	"path"
	"sync"
	"time"
)

func CheckProjects(s *store.Store) {
	wg := new(sync.WaitGroup)
	for _, source := range s.SourcePath {
		wg.Add(1)
		go func(source string) {
			rootScanner(source, s.GetProjects(), s.Logger, wg)
			wg.Done()
		}(source)
	}
	wg.Wait()

	projects := make(map[string]*store.Project)
	for _, project := range s.GetProjects() {
		if project.LastChange.Add(s.GracePeriod).Before(time.Now()) {
			project.Logger.Infof("Project %s was not touched in %v ", project.ProjectName, durafmt.Parse(time.Now().Sub(project.LastChange)).LimitFirstN(2))
			wg.Add(1)
			go scan(project, wg)
			projects[project.ProjectName] = project
		}
	}
	wg.Wait()
	for _, project := range projects {
		if project.LastChange.Add(s.GracePeriod).Before(time.Now()) {
			project.ShouldMove = true
			project.Logger.Infof("Marked Project to be moved")
		}
	}
	//TODO: inform users that projects will be moved
}

func scan(p *store.Project, group *sync.WaitGroup) {
	defer group.Done()
	log := p.Logger
	log.Infof("Scanning %s", p.ProjectName)
	lastModified := time.Time{}
	err := godirwalk.Walk(p.TargetPath, &godirwalk.Options{
		Callback: func(osPathname string, de *godirwalk.Dirent) error {
			fi, err := os.Stat(osPathname)
			if err != nil {
				log.Errorf("Error while getting Stats on %s: %v", osPathname, err)
				return err
			}
			modtime := fi.ModTime()
			if modtime.After(lastModified) {
				lastModified = modtime
			}
			return nil
		},
		Unsorted: true, // (optional) set true for faster yet non-deterministic enumeration (see godoc)
	})
	if err != nil {
		log.Errorf("Error scanning %s: %v", p.ProjectName, err)
	}
	log.Debugf("Found last modification time %v", lastModified)
	p.LastChange = lastModified
}

func NewProject(target, projectName string, createdAt time.Time, logger *nlog.Logger, wg *sync.WaitGroup) *store.Project {
	projectLogger := logger.New("Project", nlog.Data{"Project": projectName})
	project := &store.Project{TargetPath: target, Created: createdAt, ProjectName: projectName, Logger: projectLogger}
	wg.Add(1)
	go scan(project, wg)
	return project
}

func InitScanner(s *store.Store) {
	s.Logger.Infof("Initializing scanner")
	projects := s.GetProjects()
	wg := new(sync.WaitGroup)

	for _, source := range s.SourcePath {
		wg.Add(1)
		go func(source string) {
			rootScanner(source, projects, s.Logger, wg)
			wg.Done()
		}(source)
	}
	wg.Wait()

	s.Logger.Infof("Found %d projects", len(projects))
	scheduler := gocron.NewScheduler(time.Local)
	_, err := scheduler.Every(1).Day().At("08:00:00").Do(func() {
		CheckProjects(s)
	})
	if err != nil {
		s.Logger.Errorf("Failed to create Scheduler", err)
	}
	scheduler.StartImmediately()
	scheduler.StartAsync()
	s.Scheduler = scheduler
}

func rootScanner(rootPath string, projects map[string]*store.Project, logger *nlog.Logger, wg *sync.WaitGroup) {
	resorts, err := ioutil.ReadDir(rootPath)
	if err != nil {
		logger.Errorf("Error reading directory %s: %v", rootPath, err)
		return
	}
	re := regroup.MustCompile(`(?P<date>\d{4}-\d{2}-\d{2})_(?P<name>.*$)`)
	for _, r := range resorts {
		if r.IsDir() && r.Name()[0] != '.' {
			resort := path.Join(rootPath, r.Name())
			logger.Debugf("Found directory: %s", resort)
			projectsDir, err := ioutil.ReadDir(resort)
			if err != nil {
				logger.Errorf("Error reading directory %s: %v", projects, err)
				return
			}
			for _, p := range projectsDir {
				if p.IsDir() {
					_, mapContainsProject := projects[p.Name()]
					if mapContainsProject {
						continue
					}
					projectPath := path.Join(resort, p.Name())
					matches, err := re.Groups(p.Name())
					if err != nil {
						logger.Debugf("Failed to Match %s: %v", p.Name(), err)
						continue
					}
					logger.Debugf("Matches: %+v", matches)
					if len(matches) != 0 {
						createdAtString := matches["date"]
						projectName := matches["name"]
						logger.Infof("Found project: %s", projectName)

						createdAt, err := time.Parse("2006-01-02", createdAtString)
						if err != nil {
							logger.Errorf("Failed to parse date %s: %v, Using Null-Time for %s", createdAtString, err, p.Name())
							createdAt = time.Time{}
						}
						scanner := NewProject(projectPath, projectName, createdAt, logger, wg)
						projects[p.Name()] = scanner
					} else {
						logger.Debugf("Ignoring Directory %s", projectPath)
					}
				}
			}
		}
	}
	return
}