From dcf6ee9a105f588dfc4b10f502d8ce3b94be9896 Mon Sep 17 00:00:00 2001
From: Farhan Khan <khanzf@gmail.com>
Date: Sun, 19 Dec 2021 21:36:10 -0500
Subject: [PATCH] Adding trending words to restapi

---
 restapi/restapi.go | 74 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 67 insertions(+), 7 deletions(-)

diff --git a/restapi/restapi.go b/restapi/restapi.go
index 9ae7eca..cf3c7ad 100644
--- a/restapi/restapi.go
+++ b/restapi/restapi.go
@@ -12,26 +12,30 @@ import (
 	"github.com/jackc/pgx/v4"
 )
 
-var trendsText string
+var metricsText string
 
 func enableCors(w *http.ResponseWriter) {
 	(*w).Header().Set("Access-Control-Allow-Origin", "*")
 }
 
 func runMetrics() {
-	hashtagtotal := runTrendingMetrics()
+	hashtagtotal := runTrendingHashtags()
+	wordstotal := runTrendingWords()
+
 	totalJson := make(map[string]interface{})
 	totalJson["hashtags"] = hashtagtotal
+	totalJson["words"] = wordstotal
+	totalJson["datetime"] = time.Now().UTC()
 
 	data, err := json.Marshal(totalJson)
 	if err != nil {
 		log.Fatalf("error marshaling combined activity: %v\n", err)
 	}
 
-	trendsText = string(data)
+	metricsText = string(data)
 }
 
-func runTrendingMetrics() map[string]interface{} {
+func runTrendingHashtags() map[string]interface{} {
 	sql := `SELECT UNNEST(activities.hashtags) as hashtags, count(actors.id) 
 from activities 
 LEFT JOIN actors ON activities.document->>'attributedTo'=actors.document->>'id' 
@@ -65,16 +69,70 @@ GROUP BY hashtags ORDER BY count DESC LIMIT 20;`
 
 	hashtagtotal := make(map[string]interface{});
 	hashtagtotal["count"] = hashcount
-	hashtagtotal["datetime"] = time.Now().UTC()
 	hashtagtotal["items"] = hashtagitems
 
 	return hashtagtotal
 }
 
+func runTrendingWords() map[string]interface{} {
+	sql := `WITH popular_words AS (
+	select word FROM ts_stat(
+		'
+		SELECT to_tsvector(''simple'', normalized) FROM activities
+		LEFT JOIN actors ON activities.document->>''attributedTo''=actors.document->>''id''
+                WHERE activities.identifiedat > current_timestamp - interval ''60 minutes''
+		AND actors.bot=false
+		'
+	)
+	WHERE length(word) > 3
+	AND NOT word in (SELECT word FROM stopwords)
+ORDER BY ndoc DESC LIMIT 100)
+
+SELECT concat_ws(' ', a1.word, a2.word) phrase, count(*)
+FROM popular_words AS a1
+CROSS JOIN popular_words AS a2
+CROSS JOIN activities
+WHERE normalized ilike format('%%%s %s%%', a1.word, a2.word)
+AND identifiedat > current_timestamp - interval '60 minutes'
+GROUP BY 1
+HAVING count(*) > 1
+ORDER BY 2 DESC LIMIT 20;
+`
+	rows, err := pool.Query(context.Background(), sql)
+	if err != nil {
+		panic(err)
+	}
+
+	trendingitems := make([]interface{}, 0);
+	trendingcount := 0
+	for rows.Next() {
+		var trendingword string
+		var count int
+
+		err = rows.Scan(&trendingword, &count)
+		if err != nil {
+			panic(err)
+		}
+
+		trendingitem := make(map[string]interface{})
+		trendingitem["trending"] = trendingword
+		trendingitem["count"] = count
+		trendingitems = append(trendingitems, trendingitem)
+		trendingcount = trendingcount + 1
+	}
+	rows.Close()
+
+	trendingwordtotal := make(map[string]interface{});
+	trendingwordtotal["count"] = trendingcount
+	trendingwordtotal["items"] = trendingitems
+
+	return trendingwordtotal
+}
+
 // GET handlers
 func getTrending(w http.ResponseWriter, r *http.Request) {
 	enableCors(&w)
-	fmt.Fprintf(w, "%s", trendsText)
+	fmt.Fprintf(w, "%s", metricsText)
 }
 
 func getSearch(w http.ResponseWriter, r *http.Request) {
@@ -159,10 +217,12 @@ func getSearch(w http.ResponseWriter, r *http.Request) {
 func main() {
 	pool = getDbPool()
 
+	metricsText = "[]"
+
 	go func() {
 		for {
 			runMetrics()
-			time.Sleep(30 * time.Second)
+			time.Sleep(10 * time.Minute)
 		}
 	}()