diff --git a/main.go b/main.go index 5fd65b3..59e8454 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,26 @@ package main import ( + "flag" + "rodbiren.ddns.net/git/robviren/datahop/runner" ) func main() { - datahop := runner.NewRunner() - datahop.RunAll() - defer datahop.Close() + justOpen := false + + flag.BoolVar(&justOpen, "open", false, "Just opens the browser") + flag.Parse() + + if justOpen { + datahop := runner.NewRunner() + datahop.JustOpen() + defer datahop.Close() + } else { + datahop := runner.NewRunner() + datahop.RunAll() + defer datahop.Close() + } } diff --git a/runner/instagram.go b/runner/instagram.go index d54ae32..205e657 100644 --- a/runner/instagram.go +++ b/runner/instagram.go @@ -116,7 +116,11 @@ func (r *Runner) GetIntsaEgagement(urls []string) []InstaPost { fmt.Print(".") err = r.page.Navigate("https://www.instagram.com" + url) } - r.page.MustWaitLoad() + err = r.page.Timeout(5 * time.Second).WaitLoad() + for err != nil { + fmt.Print(".") + err = r.page.Timeout(5 * time.Second).WaitLoad() + } r.Scroll(10, 250) data := r.page.MustEval(instagram_post_data).JSON("", "") @@ -145,8 +149,7 @@ func (r *Runner) GetIntsaEgagement(urls []string) []InstaPost { if res.Views != 0 { res.Type = "Video" - //res.Engagement = res.Views + res.Comments - res.Engagement = 0 + res.Engagement = res.Likes + res.Comments } else { res.Type = "Image" res.Engagement = res.Likes + res.Comments diff --git a/runner/js/facebook_data.js b/runner/js/facebook_data.js index 8a2cb3c..a1d8a79 100644 --- a/runner/js/facebook_data.js +++ b/runner/js/facebook_data.js @@ -93,12 +93,15 @@ posts = [] for (var i = 0; i < arts.length; i++) { var timestamp = getTimeStamp(arts[i].getElementsByTagName("abbr")[0].innerText) + console.log(timestamp) //Not accurate var body = arts[i].innerText var url = getURL(arts[i]) var engagement = getEngagement(arts[i]) if(url){ - posts.push({ timestamp: timestamp.toISOString(), body: body, engagement: engagement, url: url }) + try { + posts.push({ timestamp: timestamp.toISOString(), body: body, engagement: engagement, url: url }) + } catch(e) {} } } diff --git a/runner/scroll.go b/runner/scroll.go index 86b2af7..167e79b 100644 --- a/runner/scroll.go +++ b/runner/scroll.go @@ -2,12 +2,21 @@ package runner import ( _ "embed" + "fmt" + "time" ) //go:embed js/scroll.js var scroll string func (r *Runner) Scroll(limit int, timeout int) { - r.page.MustEval(scroll, limit, timeout) - r.page.MustWaitLoad() + _, err := r.page.Eval(scroll, limit, timeout) + if err != nil { + panic("Scroll Error") + } + err = r.page.Timeout(5 * time.Second).WaitLoad() + for err != nil { + fmt.Println("scroll") + err = r.page.Timeout(5 * time.Second).WaitLoad() + } }