diff --git a/src/main.go b/src/main.go index 2900952..99a1f7c 100644 --- a/src/main.go +++ b/src/main.go @@ -3,7 +3,9 @@ package main import ( "database/sql" "embed" + "flag" "fmt" + "os" _ "github.com/mattn/go-sqlite3" @@ -15,16 +17,43 @@ import ( //go:embed migrations/*.sql var migrations embed.FS +var host = flag.String("host", "127.0.0.1:8000", "Set the web server host address.") + +const POSITIONAL_ARGS_HELP = ` +Valid commands: + runserver: Runs the web server. + scrape: Runs the web scraper. +` + func main() { + flag.Parse() + + if len(flag.Args()) == 0 { + fmt.Print(POSITIONAL_ARGS_HELP) + os.Exit(1) + } + + cmd := flag.Args()[0] + + client := NewScraperClient() queries, err := InitDb("db.sqlite") if err != nil { fmt.Println("Error initializing database:", err) return } - err = RunServer("0.0.0.0:3001", queries) - if err != nil { - fmt.Println("Error starting web server:", err) + switch cmd { + case "runserver": + go ScrapeEventsLoop(queries, client, true) + err = RunServer(*host, queries) + if err != nil { + fmt.Println("Error starting web server:", err) + } + case "scrape": + ScrapeEvents(queries, client, true) + default: + fmt.Print(POSITIONAL_ARGS_HELP) + os.Exit(1) } } diff --git a/src/orchestrator.go b/src/orchestrator.go index 4a6b260..7d914e3 100644 --- a/src/orchestrator.go +++ b/src/orchestrator.go @@ -9,6 +9,11 @@ import ( type Scraper func(ClientScraper) (*[]*event.Event, error) +func ScrapeEventsLoop(q *event.Queries, client ClientScraper, tapology bool) { + time.Sleep(time.Duration(1) * time.Hour) + ScrapeEvents(q, client, tapology) +} + func ScrapeEvents(q *event.Queries, client ClientScraper, tapology bool) { events := []*event.Event{} scrapers := []Scraper{ @@ -65,7 +70,7 @@ func UpdateTapology(q *event.Queries, client ClientScraper, events *[]*event.Eve } ff.Link = link } - time.Sleep(time.Duration(5_000_000_000)) + time.Sleep(time.Duration(5) * time.Second) } else { ff.Link = tapology.Url }