farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added normalization

Browse Source
This commit is contained in:
farhan 2020-12-25 05:45:08 +00:00
parent 88c074f76b
commit eafb3b9318
1 changed files with 4 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
retrieve.go
View File

@ -9,6 +9,7 @@ import (
"time" "time"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"html"
) )
type ImageType struct { type ImageType struct {
@ -127,13 +128,6 @@ func check_post(uri string) (PostJson, error) {
return postjson, nil return postjson, nil
} }
/*
err = json.NewDecoder(resp.Body).Decode(&postjson)
if err != nil {
return postjson, err
}
*/
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return postjson, nil return postjson, nil
@ -156,6 +150,9 @@ func check_post(uri string) (PostJson, error) {
} }
check_user(postjson.AttributedTo) // This must be done BEFORE the `INSERT INTO posts` below check_user(postjson.AttributedTo) // This must be done BEFORE the `INSERT INTO posts` below
postjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(postjson.Content)))
spaceReg.ReplaceAllString(postjson.normalized, " ")
connrequest = ConnRequest{} connrequest = ConnRequest{}
connrequest.conn = make(chan *pgxpool.Conn) connrequest.conn = make(chan *pgxpool.Conn)
connrequest.b = make(chan bool) connrequest.b = make(chan bool)