From eafb3b9318caa1e4e72c5f63a12bf87b4f99eb7a Mon Sep 17 00:00:00 2001
From: Farhan Khan <farhan@farhan.codes>
Date: Fri, 25 Dec 2020 05:45:08 +0000
Subject: [PATCH] added normalization

---
 retrieve.go | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/retrieve.go b/retrieve.go
index a805e91..ae4810b 100644
--- a/retrieve.go
+++ b/retrieve.go
@@ -9,6 +9,7 @@ import (
 	"time"
 	"io/ioutil"
 	"net/http"
+	"html"
 )
 
 type ImageType struct {
@@ -127,13 +128,6 @@ func check_post(uri string) (PostJson, error) {
 		return postjson, nil
 	}
 
-/*
-	err = json.NewDecoder(resp.Body).Decode(&postjson)
-	if err != nil {
-		return postjson, err
-	}
-*/
-
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return postjson, nil
@@ -156,6 +150,9 @@ func check_post(uri string) (PostJson, error) {
 	}
 	check_user(postjson.AttributedTo) // This must be done BEFORE the `INSERT INTO posts` below
 
+	postjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(postjson.Content)))
+	spaceReg.ReplaceAllString(postjson.normalized, " ")
+
 	connrequest = ConnRequest{}
 	connrequest.conn = make(chan *pgxpool.Conn)
 	connrequest.b = make(chan bool)